diff --git a/be/src/exprs/function/cast/function_cast.cpp b/be/src/exprs/function/cast/function_cast.cpp index 027bdc4db6cf17..3b647399805304 100644 --- a/be/src/exprs/function/cast/function_cast.cpp +++ b/be/src/exprs/function/cast/function_cast.cpp @@ -17,6 +17,7 @@ #include +#include "core/column/column_varbinary.h" #include "core/data_type/data_type_agg_state.h" #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_number.h" // IWYU pragma: keep @@ -26,6 +27,7 @@ #include "core/data_type/data_type_variant.h" #include "core/data_type/data_type_variant_v2.h" #include "core/data_type/primitive_type.h" +#include "core/value/uuid_value.h" #include "exprs/function/cast/cast_to_array.h" #include "exprs/function/cast/cast_to_jsonb.h" #include "exprs/function/cast/cast_to_map.h" @@ -65,7 +67,26 @@ WrapperType create_varbinary_wrapper(const DataTypePtr& from_type_untyped) { return cast_from_string_to_generic; } - return CastWrapper::create_unsupport_wrapper("Cast to Varbinary only support from String type"); + if (from_type_untyped->get_primitive_type() == TYPE_UUID) { + return [](FunctionContext*, Block& block, const ColumnNumbers& arguments, uint32_t result, + size_t rows, const NullMap::value_type*) { + const auto& source = + assert_cast(*block.get_by_position(arguments[0]).column); + auto binary = ColumnVarbinary::create(); + binary->reserve(rows); + for (size_t i = 0; i < rows; ++i) { + // File Scanner V2 decodes UUIDs before mapping Hive BINARY columns. Recreate the + // file's big-endian bytes, not the host integer layout or canonical UUID text. + const auto bytes = UUIDValue::to_big_endian(source.get_data()[i]); + binary->insert_data(reinterpret_cast(bytes.data()), bytes.size()); + } + block.get_by_position(result).column = std::move(binary); + return Status::OK(); + }; + } + + return CastWrapper::create_unsupport_wrapper( + "Cast to Varbinary only support from String or UUID type"); } WrapperType prepare_unpack_dictionaries(FunctionContext* context, const DataTypePtr& from_type, diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 43621c30fca490..12e89b3f983085 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -1736,13 +1736,14 @@ DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) { case orc::TypeKind::STRING: return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true); case orc::TypeKind::BINARY: - if (_scan_params.__isset.enable_mapping_varbinary && - _scan_params.enable_mapping_varbinary) { + // Logical UUID annotations take precedence over the default for raw binary fields. + if (orc_type->hasAttributeKey("doris.logical_type") && + orc_type->getAttributeValue("doris.logical_type") == "uuid") { + return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_UUID, true); + } else if (_scan_params.__isset.enable_mapping_varbinary && + _scan_params.enable_mapping_varbinary) { return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_VARBINARY, true); - } else if (orc_type->hasAttributeKey("doris.logical_type") && - orc_type->getAttributeValue("doris.logical_type") == "uuid") { - return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_UUID, true); } else { return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true); } @@ -2414,6 +2415,12 @@ Status OrcReader::_orc_column_to_doris_column( MutableColumnPtr data_column; if (orc_column_type != nullptr) { auto src_type = convert_to_doris_type(orc_column_type); + if (src_type->get_primitive_type() == TYPE_UUID && + data_type->get_primitive_type() == TYPE_VARBINARY) { + // Hive BINARY keeps the file bytes even when the ORC carrier has a UUID annotation. + // Decode that carrier directly, including recursively visited complex-type children. + src_type = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, true); + } bool is_dict_filter_col = false; for (const std::pair& dict_col : _dict_filter_cols) { if (col_name == dict_col.first) { diff --git a/be/src/format/parquet/parquet_column_convert.cpp b/be/src/format/parquet/parquet_column_convert.cpp index 9637ff6a6bb740..fbafb58af799d5 100644 --- a/be/src/format/parquet/parquet_column_convert.cpp +++ b/be/src/format/parquet/parquet_column_convert.cpp @@ -261,9 +261,10 @@ std::unique_ptr PhysicalToLogicalConverter::get_conv } else if (is_parquet_native_type(src_logical_primitive)) { bool is_string_logical_type = is_string_type(src_logical_primitive); if (is_string_logical_type && src_physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY) { - if (parquet_schema.logicalType.__isset.UUID && !preserve_binary_uuid) { - // Ordinary Parquet scans expose canonical text, as in File Scanner V2. - // Iceberg retains its raw 16-byte STRING carrier for equality deletes/defaults. + if (parquet_schema.logicalType.__isset.UUID && !preserve_binary_uuid && + dst_logical_type->get_primitive_type() != TYPE_VARBINARY) { + // Inferred STRING fields expose canonical text. Explicit BINARY targets and + // Iceberg's equality-delete carrier must retain the original 16 bytes. if (parquet_schema.type_length != UUIDValue::BINARY_LENGTH) { physical_converter = std::make_unique(src_physical_type, src_logical_type); diff --git a/be/src/format/parquet/schema_desc.cpp b/be/src/format/parquet/schema_desc.cpp index 09921f52496ccd..09dda646fa5007 100644 --- a/be/src/format/parquet/schema_desc.cpp +++ b/be/src/format/parquet/schema_desc.cpp @@ -268,7 +268,8 @@ std::pair FieldDescriptor::convert_to_doris_type( tparquet::LogicalType logicalType, bool nullable) { std::pair ans = {std::make_shared(), false}; bool& is_type_compatibility = ans.second; - if (logicalType.__isset.STRING) { + // ENUM is UTF-8 text, not an unannotated binary payload. + if (logicalType.__isset.STRING || logicalType.__isset.ENUM) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); } else if (logicalType.__isset.DECIMAL) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_DECIMAL128I, nullable, @@ -316,12 +317,8 @@ std::pair FieldDescriptor::convert_to_doris_type( } else if (logicalType.__isset.JSON) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); } else if (logicalType.__isset.UUID) { - if (_enable_mapping_varbinary) { - ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable, -1, - -1, 16); - } else { - ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); - } + // The raw-binary default must not override UUID's existing canonical-text contract. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); } else if (logicalType.__isset.FLOAT16) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); } else { @@ -336,6 +333,7 @@ std::pair FieldDescriptor::convert_to_doris_type( bool& is_type_compatibility = ans.second; switch (physical_schema.converted_type) { case tparquet::ConvertedType::type::UTF8: + case tparquet::ConvertedType::type::ENUM: ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); break; case tparquet::ConvertedType::type::DECIMAL: diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index b6d4e46c710abd..615beaf233d136 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -1131,11 +1131,12 @@ DataTypePtr OrcReader::_convert_to_doris_type(const ::orc::Type& type) const { data_type = std::make_shared(); break; case ::orc::TypeKind::BINARY: - if (_enable_mapping_varbinary) { - data_type = std::make_shared(); - } else if (type.hasAttributeKey("doris.logical_type") && - type.getAttributeValue("doris.logical_type") == "uuid") { + // A logical UUID remains a UUID when the raw-binary default is enabled. + if (type.hasAttributeKey("doris.logical_type") && + type.getAttributeValue("doris.logical_type") == "uuid") { data_type = std::make_shared(); + } else if (_enable_mapping_varbinary) { + data_type = std::make_shared(); } else { data_type = std::make_shared(); } diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp index 6a21ce165cd4b4..93adc748feca20 100644 --- a/be/src/format_v2/parquet/native_schema_desc.cpp +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -891,14 +891,9 @@ std::pair NativeFieldDescriptor::convert_to_doris_type( ans.first = DataTypeFactory::instance().create_data_type( TYPE_DATETIMEV2, nullable, 0, logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); } else if (logicalType.__isset.UUID) { - // Keep binary UUID carriers consistent with table defaults and other physical formats. - // Ordinary Parquet scans still expose the native UUID unless VARBINARY was requested. - auto uuid_type = TYPE_UUID; - if (_enable_mapping_varbinary) { - uuid_type = TYPE_VARBINARY; - } else if (_preserve_binary_uuid) { - uuid_type = TYPE_STRING; - } + // UUID is a logical type, so the raw-binary default must not override it. + // Preserve table-requested raw carriers for Iceberg equality deletes and defaults. + auto uuid_type = _preserve_binary_uuid ? TYPE_STRING : TYPE_UUID; ans.first = DataTypeFactory::instance().create_data_type(uuid_type, nullable, -1, -1, 16); } else if (logicalType.__isset.FLOAT16) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); diff --git a/be/test/exprs/function/cast/cast_uuid_binary_test.cpp b/be/test/exprs/function/cast/cast_uuid_binary_test.cpp new file mode 100644 index 00000000000000..6af7065aacb1a1 --- /dev/null +++ b/be/test/exprs/function/cast/cast_uuid_binary_test.cpp @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_nullable.h" +#include "core/column/column_varbinary.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_uuid.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/value/uuid_value.h" +#include "exprs/function/simple_function_factory.h" +#include "exprs/function_context.h" +#include "gtest/gtest.h" +#include "runtime/runtime_state.h" + +namespace doris { + +TEST(UuidBinaryCastTest, PreservesFileBytesAndNestedNulls) { + for (bool strict : {false, true}) { + RuntimeState state; + for (bool nested : {false, true}) { + const std::vector bytes = {std::string(16, '\0'), std::string(16, '\xff'), + std::string("\x80", 1) + std::string(15, '\0'), + "abcDeFGhijkLmnOp"}; + auto values = ColumnUUID::create(); + auto nulls = ColumnUInt8::create(); + for (const auto& value : bytes) { + values->insert_value( + UUIDValue::from_big_endian(reinterpret_cast(value.data()))); + nulls->insert_value(0); + } + values->insert_default(); + nulls->insert_value(1); + ColumnPtr source = ColumnNullable::create(std::move(values), std::move(nulls)); + DataTypePtr source_type = make_nullable(std::make_shared()); + DataTypePtr target_type = make_nullable(std::make_shared()); + if (nested) { + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(3); + offsets->insert_value(3); + offsets->insert_value(5); + source = ColumnArray::create(source, std::move(offsets)); + source_type = std::make_shared(source_type); + target_type = std::make_shared(target_type); + } + auto context = FunctionContext::create_context(&state, {}, {}); + context->set_enable_strict_mode(strict); + Block block; + block.insert({source, source_type, "uuid"}); + auto function = SimpleFunctionFactory::instance().get_function( + "CAST", {{source, source_type, "uuid"}}, target_type); + ASSERT_NE(function, nullptr); + const int result_id = block.columns(); + block.insert({nullptr, target_type, "binary"}); + auto status = function->execute(context.get(), block, {0}, result_id, source->size()); + ASSERT_TRUE(status.ok()) << status; + ColumnPtr result = block.get_by_position(result_id).column; + if (nested) { + const auto& array = assert_cast(*result); + EXPECT_EQ(array.get_offsets(), + assert_cast(*source).get_offsets()); + result = array.get_data_ptr(); + } + const auto& nullable = assert_cast(*result); + EXPECT_EQ(nullable.size(), 5); + for (size_t i = 0; i < bytes.size(); ++i) { + EXPECT_FALSE(nullable.is_null_at(i)); + EXPECT_EQ(nullable.get_nested_column().get_data_at(i).to_string(), bytes[i]); + } + EXPECT_TRUE(nullable.is_null_at(4)); + } + } +} + +} // namespace doris diff --git a/be/test/format/parquet/parquet_column_convert_test.cpp b/be/test/format/parquet/parquet_column_convert_test.cpp index 563a1795d63301..49758c281721ae 100644 --- a/be/test/format/parquet/parquet_column_convert_test.cpp +++ b/be/test/format/parquet/parquet_column_convert_test.cpp @@ -63,11 +63,14 @@ TEST(ParquetColumnConvertTest, UuidTextAndBinaryCarriersPreserveNullsAcrossBatch "00000000-0000-0000-0000-000000000000", "7f7f7f7f-7f7f-7f7f-7f7f-7f7f7f7f7f7f", "80000000-0000-0000-0000-000000000000", "ffffffff-ffff-ffff-ffff-ffffffffffff", "61626344-6546-4768-696a-6b4c6d6e4f70", "00000000-0000-0000-0000-000000000000"}; - for (const bool preserve_binary : {false, true}) { + for (int carrier = 0; carrier < 3; ++carrier) { + const bool preserve_binary = carrier == 1; + const auto dst_type = DataTypeFactory::instance().create_data_type( + carrier == 2 ? TYPE_VARBINARY : TYPE_STRING, true); auto converter = PhysicalToLogicalConverter::get_converter( - &field, field.data_type, field.data_type, nullptr, false, preserve_binary); + &field, field.data_type, dst_type, nullptr, false, preserve_binary); ASSERT_TRUE(converter->support()); - ColumnPtr dst = field.data_type->create_column(); + ColumnPtr dst = dst_type->create_column(); for (size_t batch = 0; batch < 2; ++batch) { auto values = ColumnFixedLengthObject::create(16); auto nulls = ColumnUInt8::create(); @@ -76,14 +79,14 @@ TEST(ParquetColumnConvertTest, UuidTextAndBinaryCarriersPreserveNullsAcrossBatch nulls->insert_value(i == 5); } ColumnPtr src = ColumnNullable::create(std::move(values), std::move(nulls)); - ASSERT_TRUE(converter->convert(src, field.data_type, field.data_type, dst, false).ok()); + ASSERT_TRUE(converter->convert(src, field.data_type, dst_type, dst, false).ok()); EXPECT_EQ(dst->size(), (batch + 1) * 3); } const auto& nullable = assert_cast(*dst); for (size_t i = 0; i < raw_values.size(); ++i) { EXPECT_EQ(nullable.get_null_map_data()[i], i == 5); EXPECT_EQ(nullable.get_nested_column().get_data_at(i).to_string(), - preserve_binary ? raw_values[i] : text_values[i]); + carrier != 0 ? raw_values[i] : text_values[i]); } } } diff --git a/be/test/format/parquet/parquet_reader_test.cpp b/be/test/format/parquet/parquet_reader_test.cpp index 22d669e0f4a9be..6c6c112d7f8978 100644 --- a/be/test/format/parquet/parquet_reader_test.cpp +++ b/be/test/format/parquet/parquet_reader_test.cpp @@ -369,14 +369,14 @@ TEST_F(ParquetReaderTest, normal) { delete p_reader; } -TEST_F(ParquetReaderTest, uuid_text_and_varbinary_cached_mappings) { - // Reuse the same footer cache with both mappings, then repeat each cache lookup. +TEST_F(ParquetReaderTest, uuid_annotation_preserves_text_across_mapping_flags) { + // Changing the raw-binary option must not change a cached UUID field into VARBINARY. for (const bool mapping : {false, true, false, true}) { TDescriptorTable t_desc_table; TTableDescriptor t_table_desc; std::vector table_column_names = {"id", "col1"}; - std::vector table_column_types = { - TPrimitiveType::INT, mapping ? TPrimitiveType::VARBINARY : TPrimitiveType::STRING}; + std::vector table_column_types = {TPrimitiveType::INT, + TPrimitiveType::STRING}; create_table_desc(t_desc_table, t_table_desc, table_column_names, table_column_types); DescriptorTbl* desc_tbl; ObjectPool obj_pool; @@ -438,20 +438,11 @@ TEST_F(ParquetReaderTest, uuid_text_and_varbinary_cached_mappings) { } auto col = block->safe_get_by_position(1).column; auto nullable_column = assert_cast(col.get()); - if (mapping) { - auto varbinary_column = assert_cast( - nullable_column->get_nested_column_ptr().get()); - auto& data = varbinary_column->get_data(); - EXPECT_EQ(data[0].dump_hex(), "0x550E8400E29B41D4A716446655440000"); - EXPECT_EQ(data[1].dump_hex(), "0x123E4567E89B12D3A456426614174000"); - EXPECT_EQ(data[2].dump_hex(), "0x00000000000000000000000000000000"); - } else { - const auto& strings = - assert_cast(nullable_column->get_nested_column()); - EXPECT_EQ(strings.get_data_at(0).to_string(), "550e8400-e29b-41d4-a716-446655440000"); - EXPECT_EQ(strings.get_data_at(1).to_string(), "123e4567-e89b-12d3-a456-426614174000"); - EXPECT_EQ(strings.get_data_at(2).to_string(), "00000000-0000-0000-0000-000000000000"); - } + const auto& strings = + assert_cast(nullable_column->get_nested_column()); + EXPECT_EQ(strings.get_data_at(0).to_string(), "550e8400-e29b-41d4-a716-446655440000"); + EXPECT_EQ(strings.get_data_at(1).to_string(), "123e4567-e89b-12d3-a456-426614174000"); + EXPECT_EQ(strings.get_data_at(2).to_string(), "00000000-0000-0000-0000-000000000000"); } } diff --git a/be/test/format/parquet/parquet_uuid_binary_test.cpp b/be/test/format/parquet/parquet_uuid_binary_test.cpp new file mode 100644 index 00000000000000..fa7c8e3f8eefcd --- /dev/null +++ b/be/test/format/parquet/parquet_uuid_binary_test.cpp @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include + +#include "core/column/column_array.h" +#include "core/column/column_struct.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" +#include "format/parquet/vparquet_reader.h" +#include "io/fs/file_meta_cache.h" +#include "io/fs/local_file_system.h" +#include "util/timezone_utils.h" + +namespace doris { + +TEST(ParquetUuidBinaryTest, LegacyReaderPreservesNestedBytesAcrossBatches) { + // Timestamp-semantics negotiation can force V2 even with its session flag disabled. + // Exercise the legacy reader directly to keep its nested BINARY mapping covered. + const char* source_root = std::getenv("ROOT"); + ASSERT_NE(source_root, nullptr); + const std::vector bytes = {std::string(16, '\0'), + std::string(16, '\0'), + std::string(15, '\0') + std::string(1, '\1'), + std::string(1, '\x7f') + std::string(15, '\xff'), + std::string(1, '\x80') + std::string(15, '\0'), + std::string(16, '\xff')}; + auto check_value = [&](const IColumn& column, size_t row, size_t id) { + EXPECT_EQ(column.is_null_at(row), id % 6 == 0); + if (id % 6 != 0) { + EXPECT_EQ(assert_cast(column) + .get_nested_column() + .get_data_at(row) + .to_string(), + bytes[id % 6]); + } + }; + const auto binary = make_nullable(std::make_shared()); + const DataTypes types = { + make_nullable(std::make_shared()), binary, + make_nullable(std::make_shared(binary)), + make_nullable(std::make_shared(DataTypes {binary}, Strings {"k"}))}; + const std::vector names = {"id", "u", "a", "s"}; + std::unordered_map indexes = {{"id", 0}, {"u", 1}, {"a", 2}, {"s", 3}}; + cctz::time_zone zone; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("UTC", zone)); + FileMetaCache cache(1024); + for (const std::string encoding : {"plain", "dictionary"}) { + io::FileReaderSPtr file; + ASSERT_TRUE( + io::global_local_filesystem() + ->open_file( + std::string(source_root) + + "/regression-test/data/datatype_p0/uuid/test_data/uuid_" + + encoding + ".parquet", + &file) + .ok()); + TFileScanRangeParams params; + params.__set_enable_mapping_varbinary(true); + params.__set_file_type(TFileType::FILE_LOCAL); + TFileRangeDesc range; + range.__set_start_offset(0); + range.__set_size(file->size()); + range.__set_file_size(file->size()); + range.__set_path(file->path().native()); + ParquetReader reader(nullptr, params, range, 37, &zone, nullptr, nullptr, &cache); + phmap::flat_hash_map>> predicates; + auto status = reader.init_reader(names, &indexes, {}, predicates, nullptr, nullptr, nullptr, + nullptr, nullptr); + ASSERT_TRUE(status.ok()) << status; + std::unordered_map> partitions; + std::unordered_map missing; + ASSERT_TRUE(reader.set_fill_columns(partitions, missing).ok()); + bool eof = false; + size_t total = 0; + while (!eof) { + Block block; + for (size_t i = 0; i < names.size(); ++i) { + block.insert({types[i]->create_column(), types[i], names[i]}); + } + size_t rows = 0; + status = reader.get_next_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + for (size_t row = 0; row < rows; ++row) { + const size_t id = total + row; + check_value(*block.get_by_position(1).column, row, id); + const auto& array_nullable = + assert_cast(*block.get_by_position(2).column); + EXPECT_EQ(array_nullable.is_null_at(row), id % 7 == 0); + const auto& array = + assert_cast(array_nullable.get_nested_column()); + const auto begin = array.get_offsets()[static_cast(row) - 1]; + const auto end = array.get_offsets()[row]; + if (id % 7 > 1) { + ASSERT_EQ(end - begin, 2); + check_value(array.get_data(), begin, id); + EXPECT_TRUE(array.get_data().is_null_at(begin + 1)); + } else { + EXPECT_EQ(end, begin); + } + const auto& structure = + assert_cast(*block.get_by_position(3).column); + EXPECT_EQ(structure.is_null_at(row), id % 5 == 0); + if (id % 5 != 0) { + const auto& fields = + assert_cast(structure.get_nested_column()); + check_value(fields.get_column(0), row, id); + } + } + total += rows; + } + EXPECT_EQ(total, 4097); + } +} + +} // namespace doris diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 18d3c8220a625a..0953d5f2525c05 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -4772,21 +4772,21 @@ TEST_F(NewOrcReaderTest, UuidBinaryRequiresLogicalTypeAttribute) { EXPECT_EQ(remove_nullable(array_type.get_nested_type())->get_primitive_type(), TYPE_UUID); } -TEST_F(NewOrcReaderTest, BinaryMappingOverridesUuidAnnotation) { +TEST_F(NewOrcReaderTest, UuidAnnotationOverridesBinaryMapping) { auto reader = create_reader(nullptr, std::nullopt, true); auto binary = ::orc::createPrimitiveType(::orc::BINARY); EXPECT_EQ(remove_nullable(reader->_convert_to_doris_type(*binary))->get_primitive_type(), TYPE_VARBINARY); binary->setAttribute("doris.logical_type", "uuid"); EXPECT_EQ(remove_nullable(reader->_convert_to_doris_type(*binary))->get_primitive_type(), - TYPE_VARBINARY); + TYPE_UUID); auto string = ::orc::createPrimitiveType(::orc::STRING); string->setAttribute("doris.logical_type", "uuid"); EXPECT_EQ(remove_nullable(reader->_convert_to_doris_type(*string))->get_primitive_type(), TYPE_STRING); } -TEST_F(NewOrcReaderTest, BinaryMappingOverridesNestedUuidAnnotations) { +TEST_F(NewOrcReaderTest, NestedUuidAnnotationsOverrideBinaryMapping) { auto reader = create_reader(nullptr, std::nullopt, true); const auto uuid_type = [] { auto binary = ::orc::createPrimitiveType(::orc::BINARY); @@ -4796,19 +4796,19 @@ TEST_F(NewOrcReaderTest, BinaryMappingOverridesNestedUuidAnnotations) { auto list = ::orc::createListType(uuid_type()); const auto array = remove_nullable(reader->_convert_to_doris_type(*list)); const auto& array_type = assert_cast(*array); - EXPECT_EQ(remove_nullable(array_type.get_nested_type())->get_primitive_type(), TYPE_VARBINARY); + EXPECT_EQ(remove_nullable(array_type.get_nested_type())->get_primitive_type(), TYPE_UUID); auto map = ::orc::createMapType(uuid_type(), uuid_type()); const auto mapped = remove_nullable(reader->_convert_to_doris_type(*map)); const auto& map_type = assert_cast(*mapped); - EXPECT_EQ(remove_nullable(map_type.get_key_type())->get_primitive_type(), TYPE_VARBINARY); - EXPECT_EQ(remove_nullable(map_type.get_value_type())->get_primitive_type(), TYPE_VARBINARY); + EXPECT_EQ(remove_nullable(map_type.get_key_type())->get_primitive_type(), TYPE_UUID); + EXPECT_EQ(remove_nullable(map_type.get_value_type())->get_primitive_type(), TYPE_UUID); auto structure = ::orc::createStructType(); structure->addStructField("uuid", uuid_type()); const auto converted = remove_nullable(reader->_convert_to_doris_type(*structure)); const auto& struct_type = assert_cast(*converted); - EXPECT_EQ(remove_nullable(struct_type.get_elements()[0])->get_primitive_type(), TYPE_VARBINARY); + EXPECT_EQ(remove_nullable(struct_type.get_elements()[0])->get_primitive_type(), TYPE_UUID); } TEST_F(NewOrcReaderTest, AggregatePushdownReturnsCountFromFileMetadata) { diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index cf222bbe9b582e..a810510126ac78 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -2288,9 +2288,6 @@ TEST_F(NewParquetReaderTest, UuidPlainDictionaryNullableAndMappingMatrix) { ASSERT_TRUE(reader->get_schema(&schema).ok()); ASSERT_EQ(schema.size(), 1); auto expected_type = preserve_binary_uuid ? TYPE_STRING : TYPE_UUID; - if (mapping) { - expected_type = TYPE_VARBINARY; - } EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), expected_type); auto request = std::make_shared(); @@ -2313,7 +2310,7 @@ TEST_F(NewParquetReaderTest, UuidPlainDictionaryNullableAndMappingMatrix) { if (expected_null) { continue; } - if (mapping || preserve_binary_uuid) { + if (preserve_binary_uuid) { const auto value = column.get_nested_column().get_data_at(row); EXPECT_EQ(value.size, bytes.size()); diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index 97a10c3bd69aa0..58b49b17248d11 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -62,6 +62,48 @@ TEST(ParquetSchemaTest, UnannotatedInt96DoesNotAcquireTimezoneFromMappingFlag) { } } +TEST(ParquetSchemaTest, LogicalAnnotationsOverrideBinaryDefault) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(3); + tparquet::SchemaElement modern_enum; + modern_enum.__set_name("modern_enum"); + modern_enum.__set_type(tparquet::Type::BYTE_ARRAY); + modern_enum.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + modern_enum.__set_logicalType(tparquet::LogicalType()); + modern_enum.logicalType.__set_ENUM(tparquet::EnumType()); + auto legacy_enum = modern_enum; + legacy_enum.__set_name("legacy_enum"); + legacy_enum.__isset.logicalType = false; + legacy_enum.__set_converted_type(tparquet::ConvertedType::ENUM); + auto uuid = modern_enum; + uuid.__set_name("uuid"); + uuid.__set_type(tparquet::Type::FIXED_LEN_BYTE_ARRAY); + uuid.__set_type_length(16); + uuid.__set_logicalType(tparquet::LogicalType()); + uuid.logicalType.__set_UUID(tparquet::UUIDType()); + for (bool binary_mapping : {false, true}) { + const std::vector schema {root, modern_enum, legacy_enum, uuid}; + doris::FieldDescriptor legacy; + legacy.set_enable_mapping_varbinary(binary_mapping); + ASSERT_TRUE(legacy.parse_from_thrift(schema).ok()); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(legacy.get_column(i)->data_type->get_primitive_type(), TYPE_STRING); + } + NativeFieldDescriptor native; + native.set_enable_mapping_varbinary(binary_mapping); + ASSERT_TRUE(native.parse_from_thrift(schema).ok()); + EXPECT_EQ(native.get_column(0)->data_type->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(native.get_column(1)->data_type->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(native.get_column(2)->data_type->get_primitive_type(), TYPE_UUID); + NativeFieldDescriptor iceberg; + iceberg.set_enable_mapping_varbinary(binary_mapping); + iceberg.set_preserve_binary_uuid(true); + ASSERT_TRUE(iceberg.parse_from_thrift(schema).ok()); + EXPECT_EQ(iceberg.get_column(2)->data_type->get_primitive_type(), TYPE_STRING); + } +} + namespace { std::vector unshredded_variant_schema( diff --git a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java index 1e404809b7737d..2fe0c5ba5742a0 100644 --- a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java +++ b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java @@ -20,6 +20,8 @@ import org.apache.doris.common.jni.vec.ColumnValue; import org.apache.iceberg.StructLike; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; import java.math.BigDecimal; import java.math.BigInteger; @@ -38,14 +40,20 @@ public class IcebergSysTableColumnValue implements ColumnValue { private final Object fieldData; private final String timezone; + private final Type icebergType; public IcebergSysTableColumnValue(Object fieldData) { this(fieldData, DEFAULT_TIME_ZONE); } public IcebergSysTableColumnValue(Object fieldData, String timezone) { + this(fieldData, timezone, null); + } + + public IcebergSysTableColumnValue(Object fieldData, String timezone, Type icebergType) { this.fieldData = fieldData; this.timezone = timezone; + this.icebergType = icebergType; } @Override @@ -136,14 +144,22 @@ public LocalDate getDate() { @Override public LocalDateTime getDateTime() { - Instant instant = Instant.ofEpochMilli((((long) fieldData) / 1000)); - return LocalDateTime.ofInstant(instant, ZoneId.of(timezone)); + // NTZ is a wall-clock value; only timestamptz mapped to DATETIME uses the session zone. + ZoneId zone = icebergType instanceof Types.TimestampType + && !((Types.TimestampType) icebergType).shouldAdjustToUTC() + ? ZoneOffset.UTC : ZoneId.of(timezone); + return LocalDateTime.ofInstant(timestampInstant(), zone); } @Override public LocalDateTime getTimeStampTz() { - Instant instant = Instant.ofEpochMilli((((long) fieldData) / 1000)); - return LocalDateTime.ofInstant(instant, ZoneId.of("UTC")); + return LocalDateTime.ofInstant(timestampInstant(), ZoneOffset.UTC); + } + + private Instant timestampInstant() { + long micros = (long) fieldData; + return Instant.ofEpochSecond(Math.floorDiv(micros, 1_000_000L), + Math.floorMod(micros, 1_000_000L) * 1_000L); } @Override @@ -162,7 +178,8 @@ public byte[] getBytes() { public void unpackArray(List values) { List items = (List) fieldData; for (Object item : items) { - values.add(new IcebergSysTableColumnValue(item, timezone)); + values.add(new IcebergSysTableColumnValue(item, timezone, + icebergType == null ? null : icebergType.asListType().elementType())); } } @@ -170,8 +187,10 @@ public void unpackArray(List values) { public void unpackMap(List keys, List values) { Map data = (Map) fieldData; data.forEach((key, value) -> { - keys.add(new IcebergSysTableColumnValue(key, timezone)); - values.add(new IcebergSysTableColumnValue(value, timezone)); + keys.add(new IcebergSysTableColumnValue(key, timezone, + icebergType == null ? null : icebergType.asMapType().keyType())); + values.add(new IcebergSysTableColumnValue(value, timezone, + icebergType == null ? null : icebergType.asMapType().valueType())); }); } @@ -180,7 +199,8 @@ public void unpackStruct(List structFieldIndex, List value StructLike record = (StructLike) fieldData; for (Integer fieldIndex : structFieldIndex) { Object rawValue = record.get(fieldIndex, Object.class); - values.add(new IcebergSysTableColumnValue(rawValue, timezone)); + values.add(new IcebergSysTableColumnValue(rawValue, timezone, + icebergType == null ? null : icebergType.asStructType().fields().get(fieldIndex).type())); } } } diff --git a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java index b014a42706ff29..a35df84920868f 100644 --- a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java +++ b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java @@ -109,7 +109,7 @@ protected int getNext() throws IOException { for (int i = 0; i < fields.size(); i++) { SelectedField field = fields.get(i); Object value = row.get(field.sourceIndex, field.field.type().typeId().javaClass()); - ColumnValue columnValue = new IcebergSysTableColumnValue(value, timezone); + ColumnValue columnValue = new IcebergSysTableColumnValue(value, timezone, field.field.type()); appendData(i, columnValue); } rows++; diff --git a/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergSysTableColumnValueTest.java b/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergSysTableColumnValueTest.java new file mode 100644 index 00000000000000..c18d51517bba36 --- /dev/null +++ b/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergSysTableColumnValueTest.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.iceberg; + +import org.apache.doris.common.jni.vec.ColumnValue; + +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class IcebergSysTableColumnValueTest { + @Test + public void preservesMicrosAndTimestampSemantics() { + long[] micros = {-1000001L, -1L, 1L, 999L, 1000L, 1234567L}; + String[] expected = {"1969-12-31T23:59:58.999999", "1969-12-31T23:59:59.999999", + "1970-01-01T00:00:00.000001", "1970-01-01T00:00:00.000999", + "1970-01-01T00:00:00.001", "1970-01-01T00:00:01.234567"}; + for (int i = 0; i < micros.length; i++) { + LocalDateTime utc = LocalDateTime.parse(expected[i]); + for (String zone : Arrays.asList("UTC", "Asia/Shanghai")) { + ColumnValue ntz = new IcebergSysTableColumnValue(micros[i], zone, Types.TimestampType.withoutZone()); + ColumnValue tz = new IcebergSysTableColumnValue(micros[i], zone, Types.TimestampType.withZone()); + Assertions.assertEquals(utc, ntz.getDateTime()); + Assertions.assertEquals(zone.equals("UTC") ? utc : utc.plusHours(8), tz.getDateTime()); + Assertions.assertEquals(utc, tz.getTimeStampTz()); + } + } + } + + @Test + public void propagatesTypesThroughNestedContainers() { + Types.StructType struct = Types.StructType.of( + Types.NestedField.optional(1, "ts", Types.TimestampType.withoutZone()), + Types.NestedField.optional(2, "tz", Types.TimestampType.withZone())); + PartitionData record = new PartitionData(struct); + record.set(0, -1L); + record.set(1, -1L); + List fields = new ArrayList<>(); + new IcebergSysTableColumnValue(record, "Asia/Shanghai", struct) + .unpackStruct(Arrays.asList(1, 0), fields); + LocalDateTime utc = LocalDateTime.parse("1969-12-31T23:59:59.999999"); + Assertions.assertEquals(utc.plusHours(8), fields.get(0).getDateTime()); + Assertions.assertEquals(utc, fields.get(1).getDateTime()); + + List elements = new ArrayList<>(); + new IcebergSysTableColumnValue(Collections.singletonList(-1L), "Asia/Shanghai", + Types.ListType.ofOptional(3, Types.TimestampType.withoutZone())).unpackArray(elements); + Assertions.assertEquals(utc, elements.get(0).getDateTime()); + + List keys = new ArrayList<>(); + List values = new ArrayList<>(); + new IcebergSysTableColumnValue(Collections.singletonMap(-1L, -1L), "Asia/Shanghai", + Types.MapType.ofOptional(4, 5, Types.TimestampType.withoutZone(), Types.TimestampType.withZone())) + .unpackMap(keys, values); + Assertions.assertEquals(utc, keys.get(0).getDateTime()); + Assertions.assertEquals(utc.plusHours(8), values.get(0).getDateTime()); + } +} diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java index 4332c1472f2f13..1fdcc3cd12f3a9 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java @@ -78,6 +78,8 @@ protected void initializeStatement(Connection conn, JdbcDataSourceConfig config, @Override protected void setValidationQuery(HikariDataSource ds) { ds.setConnectionTestQuery("SELECT 1 FROM dual"); + // NUMBER binding rejects finite IEEE values outside NUMBER's exponent range. + ds.addDataSourceProperty("SetFloatAndDoubleUseBinary", "true"); } @Override diff --git a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/OracleSessionTimeZoneTest.java b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/OracleSessionTimeZoneTest.java index ee525d7f1e4481..f1901c4f48b9c8 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/OracleSessionTimeZoneTest.java +++ b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/OracleSessionTimeZoneTest.java @@ -18,18 +18,102 @@ package org.apache.doris.jdbc; +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.VectorColumn; + +import com.zaxxer.hikari.HikariDataSource; import oracle.jdbc.OracleConnection; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.mockito.InOrder; import org.mockito.Mockito; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; class OracleSessionTimeZoneTest { + @Test + void binaryFloatingPointBindingsAreEnabledBeforeOpeningConnections() { + OracleJdbcExecutor handler = Mockito.mock(OracleJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); + try (HikariDataSource ds = new HikariDataSource()) { + handler.setValidationQuery(ds); + Assertions.assertEquals("true", ds.getDataSourceProperties().getProperty( + OracleConnection.CONNECTION_PROPERTY_SET_FLOAT_AND_DOUBLE_USE_BINARY)); + } + } + + @Test + void binaryEncodingPreservesValuesOutsideNumberRange() throws Exception { + Assertions.assertThrows(SQLException.class, () -> new oracle.sql.NUMBER(1e200)); + for (double value : new double[] {1e200, -1e200, 1e-200, Double.MAX_VALUE, Double.MIN_VALUE}) { + byte[] encoded = new oracle.sql.BINARY_DOUBLE(value).getBytes(); + Assertions.assertEquals(value, new oracle.sql.BINARY_DOUBLE(encoded).doubleValue()); + } + for (float value : new float[] {1.25f, -1.25f, Float.MAX_VALUE, Float.MIN_VALUE}) { + byte[] encoded = new oracle.sql.BINARY_FLOAT(value).getBytes(); + Assertions.assertEquals(value, new oracle.sql.BINARY_FLOAT(encoded).floatValue()); + } + } + + @Test + @EnabledIfSystemProperty(named = "oracle.integration.url", matches = ".+") + void binaryFloatingPointInsertRoundTrip() throws Exception { + OracleJdbcExecutor handler = Mockito.mock(OracleJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); + try (HikariDataSource ds = new HikariDataSource()) { + handler.setValidationQuery(ds); + ds.setJdbcUrl(System.getProperty("oracle.integration.url")); + ds.setUsername(System.getProperty("oracle.integration.user")); + ds.setPassword(System.getProperty("oracle.integration.password")); + ds.setMaximumPoolSize(1); + try (Connection connection = ds.getConnection(); Statement ddl = connection.createStatement()) { + String table = "doris_binary_" + java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 8); + ddl.execute("CREATE TABLE " + table + " (id INT, f BINARY_FLOAT, d BINARY_DOUBLE)"); + try (PreparedStatement insert = connection.prepareStatement("INSERT INTO " + table + " VALUES (?, ?, ?)")) { + handler.preparedStatement = insert; + VectorColumn floats = Mockito.mock(VectorColumn.class); + VectorColumn doubles = Mockito.mock(VectorColumn.class); + Mockito.when(floats.getColumnPrimitiveType()).thenReturn(ColumnType.Type.FLOAT); + Mockito.when(doubles.getColumnPrimitiveType()).thenReturn(ColumnType.Type.DOUBLE); + float[] floatValues = {1.25f, -1.25f, 0.0f}; + double[] doubleValues = {1e200, -1e200, 1e-200}; + java.lang.reflect.Method insertColumn = BaseJdbcExecutor.class.getDeclaredMethod( + "insertColumn", int.class, int.class, VectorColumn.class); + insertColumn.setAccessible(true); + Mockito.when(floats.isNullAt(3)).thenReturn(true); + Mockito.when(doubles.isNullAt(3)).thenReturn(true); + for (int row = 0; row < 4; ++row) { + if (row < 3) { + Mockito.when(floats.getFloat(row)).thenReturn(floatValues[row]); + Mockito.when(doubles.getDouble(row)).thenReturn(doubleValues[row]); + } + insert.setInt(1, row); + insertColumn.invoke(handler, row, 1, floats); + insertColumn.invoke(handler, row, 2, doubles); + insert.executeUpdate(); + } + try (ResultSet rows = ddl.executeQuery("SELECT f, d FROM " + table + " ORDER BY id")) { + for (int row = 0; row < 3; ++row) { + Assertions.assertTrue(rows.next()); + Assertions.assertEquals(floatValues[row], rows.getFloat(1)); + Assertions.assertEquals(doubleValues[row], rows.getDouble(2)); + } + Assertions.assertTrue(rows.next()); + Assertions.assertNull(rows.getObject(1)); + Assertions.assertNull(rows.getObject(2)); + Assertions.assertFalse(rows.next()); + } + } finally { + ddl.execute("DROP TABLE " + table); + } + } + } + } + @Test void initializesEveryBorrowedConnectionBeforePreparingQuery() throws Exception { OracleJdbcExecutor handler = Mockito.mock(OracleJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java index 170db84b0d5da5..27a09005008610 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java @@ -208,6 +208,10 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { precision = scale; } return createDecimalOrStringType(precision, scale); + // Oracle binary floating-point types retain IEEE precision instead of becoming unsupported. + case "BINARY_FLOAT": + return Type.FLOAT; + case "BINARY_DOUBLE": case "FLOAT": return Type.DOUBLE; case "DATE": @@ -227,8 +231,6 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { return ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize()); case "NCLOB": case "BFILE": - case "BINARY_FLOAT": - case "BINARY_DOUBLE": default: return Type.UNSUPPORTED; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java index 35edd6c96babd3..e3300d8ce5b913 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java @@ -21,8 +21,19 @@ import org.apache.doris.catalog.Type; import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; +import java.sql.Types; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + public class JdbcSQLServerClient extends JdbcClient { + // TYPE_NAME of an IDENTITY column decorates the base type: "int identity", "decimal() identity", + // "numeric(18, 0) identity", "decimal(18,0) IDENTITY(1,1)". IDENTITY is only allowed on these base types. + private static final Pattern IDENTITY_TYPE_NAME = Pattern.compile( + "^(tinyint|smallint|int|bigint|decimal|numeric)\\s*(\\([^)]*\\))?\\s+identity(\\s*\\([^)]*\\))?$", + Pattern.CASE_INSENSITIVE); + protected JdbcSQLServerClient(JdbcClientConfig jdbcClientConfig) { super(jdbcClientConfig); } @@ -34,18 +45,48 @@ protected String getSchemaPatternForDatabaseNameList() { return "%"; } + /** + * The base type of an IDENTITY column's TYPE_NAME, or the name unchanged. + *

+ * The decoration is trusted only when {@code DATA_TYPE} is the code of the named base type: an alias type + * may legally be named like that ({@code CREATE TYPE dbo.[int identity] FROM varchar(10)}, or + * {@code dbo.[int alias]}), and it then has to be resolved by its code, not by the words of its name. + */ + static String identityBaseType(String typeName, int dataType) { + Matcher matcher = IDENTITY_TYPE_NAME.matcher(typeName); + if (!matcher.matches()) { + return typeName; + } + String baseType = matcher.group(1).toLowerCase(Locale.ROOT); + boolean codeMatches; + switch (baseType) { + case "tinyint": + codeMatches = dataType == Types.TINYINT; + break; + case "smallint": + codeMatches = dataType == Types.SMALLINT; + break; + case "int": + codeMatches = dataType == Types.INTEGER; + break; + case "bigint": + codeMatches = dataType == Types.BIGINT; + break; + default: + codeMatches = dataType == Types.DECIMAL || dataType == Types.NUMERIC; + break; + } + return codeMatches ? baseType : typeName; + } + @Override protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { String originSqlserverType = fieldSchema.getDataTypeName().orElse("unknown"); - // For sqlserver IDENTITY type, such as 'INT IDENTITY' - // originSqlserverType is "int identity", so we only get "int". - // For types with parameters like 'decimal(18,0) IDENTITY(1,1)', we need to extract the base type - String sqlserverType = originSqlserverType.split(" ")[0]; - - // Handle types with parentheses like decimal(18,0), varchar(50), etc. - if (sqlserverType.contains("(")) { - sqlserverType = sqlserverType.substring(0, sqlserverType.indexOf("(")); - } + // An IDENTITY column is reported as "int identity" or "decimal(18,0) identity": only the base type + // is matched below. Any other name is matched as it is: system type names are single words, and a + // user-defined alias type may be named with spaces or parentheses ("int alias") and must not be + // mistaken for the system type its name starts with. + String sqlserverType = identityBaseType(originSqlserverType, fieldSchema.getDataType()); switch (sqlserverType) { case "bit": @@ -100,6 +141,76 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { case "binary": case "varbinary": return ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize()); + case "xml": + case "sql_variant": + case "geometry": + case "geography": + case "hierarchyid": + case "json": + case "vector": + // SQL Server system types that Doris does not support. They are listed explicitly + // so that they never reach the JDBC type code fallback below. + return Type.UNSUPPORTED; + default: + return jdbcTypeCodeToDoris(fieldSchema); + } + } + + /** + * Fallback for type names that are not SQL Server system types. + *

+ * User-defined alias types ({@code CREATE TYPE dbo.my_type FROM varchar(50)}) are reported by + * {@code DatabaseMetaData.getColumns()} with {@code TYPE_NAME} set to the alias name, so they can not be + * matched by name. {@code DATA_TYPE}, {@code COLUMN_SIZE} and {@code DECIMAL_DIGITS} still describe the + * base type, so the standard {@link Types} code is used to resolve the Doris type. The mapping mirrors + * the name based one above. + *

+ * Binary codes are deliberately not mapped: mssql-jdbc also reports CLR user-defined types + * (geometry, geography, hierarchyid, ...) as {@link Types#VARBINARY}, so they can not be told apart from + * an alias over a binary type by the type code alone. Vendor specific codes stay unsupported as well. + */ + private Type jdbcTypeCodeToDoris(JdbcFieldSchema fieldSchema) { + switch (fieldSchema.getDataType()) { + case Types.BIT: + case Types.BOOLEAN: + return Type.BOOLEAN; + // SQL Server tinyint is unsigned (0 to 255), so it needs SMALLINT + case Types.TINYINT: + case Types.SMALLINT: + return Type.SMALLINT; + case Types.INTEGER: + return Type.INT; + case Types.BIGINT: + return Type.BIGINT; + case Types.REAL: + return Type.FLOAT; + case Types.FLOAT: + case Types.DOUBLE: + return Type.DOUBLE; + case Types.DECIMAL: + case Types.NUMERIC: { + // money and smallmoney are reported as DECIMAL(19,4) and DECIMAL(10,4) + int precision = fieldSchema.getColumnSize().orElse(0); + int scale = fieldSchema.getDecimalDigits().orElse(0); + return createDecimalOrStringType(precision, scale); + } + case Types.DATE: + return ScalarType.createDateV2Type(); + case Types.TIMESTAMP: { + int scale = fieldSchema.getDecimalDigits().orElse(0); + if (scale > 6) { + scale = 6; + } + return ScalarType.createDatetimeV2Type(scale); + } + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + case Types.NCHAR: + case Types.NVARCHAR: + case Types.LONGNVARCHAR: + case Types.TIME: + return ScalarType.createStringType(); default: return Type.UNSUPPORTED; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 41466e0648a718..398cbb4e02cc25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -253,12 +253,9 @@ && getTFileType() != TFileType.FILE_S3) { throw new AnalysisException("Lance format is supported only by local() and s3() TVFs"); } - // The catalog property was removed, but this TVF-only option must remain explicit because - // changing an ad-hoc TVF result schema also breaks CTAS type inference. - String enableMappingVarbinaryStr = getOrDefaultAndRemove(copiedProps, - FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY, "false"); - fileFormatProperties.enableMappingVarbinary = Boolean.parseBoolean(enableMappingVarbinaryStr); - + // Binary file fields must preserve bytes even when a legacy opt-out is supplied. + copiedProps.remove(FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY); + fileFormatProperties.enableMappingVarbinary = true; // Consume the legacy option, but let file logical types determine timezone semantics. copiedProps.remove(FileFormatConstants.PROP_ENABLE_MAPPING_TIMESTAMP_TZ); fileFormatProperties.enableMappingTimestampTz = true; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClientTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClientTest.java new file mode 100644 index 00000000000000..6e14a38e95dc26 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClientTest.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.jdbc.client; + +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Answers; +import org.mockito.Mockito; + +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; + +public class JdbcOracleClientTest { + private final JdbcOracleClient client = Mockito.mock(JdbcOracleClient.class, Answers.CALLS_REAL_METHODS); + + private static JdbcFieldSchema column(String typeName, int dataType) throws SQLException { + ResultSet resultSet = Mockito.mock(ResultSet.class); + Mockito.when(resultSet.getString("COLUMN_NAME")).thenReturn("col"); + Mockito.when(resultSet.getInt("DATA_TYPE")).thenReturn(dataType); + Mockito.when(resultSet.getString("TYPE_NAME")).thenReturn(typeName); + Mockito.when(resultSet.getInt("COLUMN_SIZE")).thenReturn(8); + Mockito.when(resultSet.getInt("DECIMAL_DIGITS")).thenReturn(0); + Mockito.when(resultSet.getInt("NULLABLE")).thenReturn(DatabaseMetaData.columnNullable); + return new JdbcFieldSchema(resultSet); + } + + @Test + public void testOracleBinaryFloatingPointTypes() throws SQLException { + Assertions.assertEquals(Type.FLOAT, + client.jdbcTypeToDoris(column("BINARY_FLOAT", Types.FLOAT))); + Assertions.assertEquals(Type.DOUBLE, + client.jdbcTypeToDoris(column("BINARY_DOUBLE", Types.DOUBLE))); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClientTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClientTest.java new file mode 100644 index 00000000000000..fa184a22218c6a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClientTest.java @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.jdbc.client; + +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Answers; +import org.mockito.Mockito; + +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; + +public class JdbcSQLServerClientTest { + + // ODBC type codes that mssql-jdbc passes through DatabaseMetaData.getColumns() unchanged + private static final int SQL_VARIANT = -150; + private static final int SQL_SS_TIMESTAMPOFFSET = -155; + + private final JdbcSQLServerClient client = Mockito.mock(JdbcSQLServerClient.class, Answers.CALLS_REAL_METHODS); + + /** + * Builds the schema of one DatabaseMetaData.getColumns() row as reported by mssql-jdbc. + * For a user-defined alias type, TYPE_NAME is the alias name while DATA_TYPE, COLUMN_SIZE + * and DECIMAL_DIGITS describe the base type. + */ + private static JdbcFieldSchema column(String typeName, int dataType, int columnSize, int decimalDigits) + throws SQLException { + ResultSet rs = Mockito.mock(ResultSet.class); + Mockito.when(rs.getString("COLUMN_NAME")).thenReturn("col"); + Mockito.when(rs.getInt("DATA_TYPE")).thenReturn(dataType); + Mockito.when(rs.getString("TYPE_NAME")).thenReturn(typeName); + Mockito.when(rs.getInt("COLUMN_SIZE")).thenReturn(columnSize); + Mockito.when(rs.getInt("DECIMAL_DIGITS")).thenReturn(decimalDigits); + Mockito.when(rs.getInt("NULLABLE")).thenReturn(DatabaseMetaData.columnNullable); + return new JdbcFieldSchema(rs); + } + + @Test + public void testAliasTypeIsResolvedByJdbcTypeCode() throws SQLException { + // CREATE TYPE dbo.customtexttype FROM varchar(50), the case reported in #67793 + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("customtexttype", Types.VARCHAR, 50, 0))); + // sysname is a built-in alias over nvarchar(128) + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("sysname", Types.NVARCHAR, 128, 0))); + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("alias_nchar", Types.NCHAR, 10, 0))); + Assertions.assertEquals(Type.STRING, + client.jdbcTypeToDoris(column("alias_text", Types.LONGVARCHAR, Integer.MAX_VALUE, 0))); + Assertions.assertEquals(Type.STRING, + client.jdbcTypeToDoris(column("alias_ntext", Types.LONGNVARCHAR, Integer.MAX_VALUE / 2, 0))); + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("alias_time", Types.TIME, 16, 7))); + // uniqueidentifier is reported as CHAR(36) + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("alias_guid", Types.CHAR, 36, 0))); + + Assertions.assertEquals(Type.BOOLEAN, client.jdbcTypeToDoris(column("alias_bit", Types.BIT, 1, 0))); + // SQL Server tinyint is unsigned, so it keeps the SMALLINT mapping of the name based path + Assertions.assertEquals(Type.SMALLINT, client.jdbcTypeToDoris(column("alias_tinyint", Types.TINYINT, 3, 0))); + Assertions.assertEquals(Type.SMALLINT, client.jdbcTypeToDoris(column("alias_smallint", Types.SMALLINT, 5, 0))); + Assertions.assertEquals(Type.INT, client.jdbcTypeToDoris(column("alias_int", Types.INTEGER, 10, 0))); + Assertions.assertEquals(Type.BIGINT, client.jdbcTypeToDoris(column("alias_bigint", Types.BIGINT, 19, 0))); + Assertions.assertEquals(Type.FLOAT, client.jdbcTypeToDoris(column("alias_real", Types.REAL, 24, 0))); + Assertions.assertEquals(Type.DOUBLE, client.jdbcTypeToDoris(column("alias_float", Types.DOUBLE, 53, 0))); + + Assertions.assertEquals(ScalarType.createDecimalV3Type(10, 2), + client.jdbcTypeToDoris(column("alias_decimal", Types.DECIMAL, 10, 2))); + Assertions.assertEquals(ScalarType.createDecimalV3Type(38, 10), + client.jdbcTypeToDoris(column("alias_numeric", Types.NUMERIC, 38, 10))); + // money is reported as DECIMAL(19,4), the same as the name based mapping produces + Assertions.assertEquals(ScalarType.createDecimalV3Type(19, 4), + client.jdbcTypeToDoris(column("alias_money", Types.DECIMAL, 19, 4))); + + Assertions.assertEquals(Type.DATEV2, client.jdbcTypeToDoris(column("alias_date", Types.DATE, 10, 0))); + Assertions.assertEquals(ScalarType.createDatetimeV2Type(3), + client.jdbcTypeToDoris(column("alias_datetime", Types.TIMESTAMP, 23, 3))); + // datetime2 defaults to 7 fractional digits, Doris supports at most 6 + Assertions.assertEquals(ScalarType.createDatetimeV2Type(6), + client.jdbcTypeToDoris(column("alias_datetime2", Types.TIMESTAMP, 27, 7))); + Assertions.assertEquals(ScalarType.createDatetimeV2Type(0), + client.jdbcTypeToDoris(column("alias_smalldatetime", Types.TIMESTAMP, 16, 0))); + } + + @Test + public void testUnknownTypesStayUnsupported() throws SQLException { + // CLR user-defined types are reported as VARBINARY, exactly like an alias over varbinary, + // so binary codes must not be resolved by the fallback + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("geometry", Types.VARBINARY, Integer.MAX_VALUE, 0))); + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("my_clr_type", Types.VARBINARY, 8000, 0))); + Assertions.assertEquals(Type.UNSUPPORTED, client.jdbcTypeToDoris(column("alias_binary", Types.BINARY, 20, 0))); + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("alias_image", Types.LONGVARBINARY, Integer.MAX_VALUE, 0))); + // vendor specific type codes + Assertions.assertEquals(Type.UNSUPPORTED, client.jdbcTypeToDoris(column("sql_variant", SQL_VARIANT, 8000, 0))); + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("alias_datetimeoffset", SQL_SS_TIMESTAMPOFFSET, 34, 7))); + // explicitly unsupported system types keep that behavior whatever type code the driver reports + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("xml", Types.LONGNVARCHAR, Integer.MAX_VALUE / 2, 0))); + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("json", Types.LONGNVARCHAR, Integer.MAX_VALUE / 2, 0))); + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("hierarchyid", Types.VARBINARY, 892, 0))); + } + + @Test + public void testAliasNamedLikeASystemTypeIsResolvedByJdbcTypeCode() throws SQLException { + // A delimited alias name may contain spaces and parentheses ([int alias], [decimal(18,0) identity]); + // it is reported as is, and the base type is still what DATA_TYPE says + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("int alias", Types.VARCHAR, 50, 0))); + Assertions.assertEquals(Type.STRING, + client.jdbcTypeToDoris(column("decimal(18,0) identity", Types.NVARCHAR, 20, 0))); + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("int identity", Types.VARCHAR, 10, 0))); + Assertions.assertEquals(Type.STRING, + client.jdbcTypeToDoris(column("bigint identity", Types.NVARCHAR, 20, 0))); + Assertions.assertEquals(ScalarType.createDatetimeV2Type(3), + client.jdbcTypeToDoris(column("varchar(50) alias", Types.TIMESTAMP, 23, 3))); + + // The IDENTITY decoration of a real system type, in the forms the driver versions report it in + Assertions.assertEquals(Type.INT, client.jdbcTypeToDoris(column("int identity", Types.INTEGER, 10, 0))); + Assertions.assertEquals(Type.BIGINT, client.jdbcTypeToDoris(column("bigint identity", Types.BIGINT, 19, 0))); + Assertions.assertEquals(Type.SMALLINT, + client.jdbcTypeToDoris(column("tinyint identity", Types.TINYINT, 3, 0))); + Assertions.assertEquals(ScalarType.createDecimalV3Type(18, 0), + client.jdbcTypeToDoris(column("decimal identity", Types.DECIMAL, 18, 0))); + Assertions.assertEquals(ScalarType.createDecimalV3Type(18, 0), + client.jdbcTypeToDoris(column("decimal() identity", Types.DECIMAL, 18, 0))); + Assertions.assertEquals(ScalarType.createDecimalV3Type(18, 0), + client.jdbcTypeToDoris(column("numeric(18, 0) identity", Types.NUMERIC, 18, 0))); + Assertions.assertEquals(ScalarType.createDecimalV3Type(18, 0), + client.jdbcTypeToDoris(column("decimal(18,0) IDENTITY(1,1)", Types.DECIMAL, 18, 0))); + } + + @Test + public void testSystemTypeNamesTakePrecedence() throws SQLException { + // the name based mapping is unchanged, the type code is only consulted for unknown names + Assertions.assertEquals(Type.SMALLINT, client.jdbcTypeToDoris(column("tinyint", Types.TINYINT, 3, 0))); + Assertions.assertEquals(Type.INT, client.jdbcTypeToDoris(column("int identity", Types.INTEGER, 10, 0))); + Assertions.assertEquals(ScalarType.createDecimalV3Type(19, 4), + client.jdbcTypeToDoris(column("money", Types.DECIMAL, 19, 4))); + Assertions.assertEquals(ScalarType.createVarbinaryType(20), + client.jdbcTypeToDoris(column("varbinary", Types.VARBINARY, 20, 0))); + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("timestamp", Types.BINARY, 8, 0))); + Assertions.assertEquals(ScalarType.createTimeStampTzType(6), + client.jdbcTypeToDoris(column("datetimeoffset", SQL_SS_TIMESTAMPOFFSET, 34, 7))); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java index a9bc964fbb1100..f08af573bef7f1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java @@ -101,24 +101,21 @@ public void testHiveParquetTimeZoneRejectsAmbiguousShortAlias() { } @Test - public void testTvfVarbinaryMappingRemainsExplicit() throws AnalysisException { - ExternalFileTableValuedFunction defaultTvf = Mockito.mock( - ExternalFileTableValuedFunction.class, Mockito.CALLS_REAL_METHODS); - Map defaultProperties = Maps.newHashMap(); - defaultProperties.put(FileFormatConstants.PROP_FORMAT, FileFormatConstants.FORMAT_PARQUET); - - defaultTvf.parseCommonProperties(defaultProperties); - - Assert.assertFalse(defaultTvf.fileFormatProperties.enableMappingVarbinary); - - ExternalFileTableValuedFunction enabledTvf = Mockito.mock( - ExternalFileTableValuedFunction.class, Mockito.CALLS_REAL_METHODS); - Map enabledProperties = Maps.newHashMap(defaultProperties); - enabledProperties.put(FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY, "true"); - - enabledTvf.parseCommonProperties(enabledProperties); - - Assert.assertTrue(enabledTvf.fileFormatProperties.enableMappingVarbinary); + public void testTvfVarbinaryMappingCannotBeDisabled() throws AnalysisException { + for (String format : new String[] {FileFormatConstants.FORMAT_PARQUET, FileFormatConstants.FORMAT_ORC}) { + for (String legacyValue : new String[] {null, "false", "true"}) { + ExternalFileTableValuedFunction tvf = Mockito.mock( + ExternalFileTableValuedFunction.class, Mockito.CALLS_REAL_METHODS); + Map properties = Maps.newHashMap(); + properties.put(FileFormatConstants.PROP_FORMAT, format); + if (legacyValue != null) { + properties.put(FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY, legacyValue); + } + Map remaining = tvf.parseCommonProperties(properties); + Assert.assertTrue(tvf.fileFormatProperties.enableMappingVarbinary); + Assert.assertFalse(remaining.containsKey(FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY)); + } + } } @Test diff --git a/regression-test/data/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.out b/regression-test/data/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.out index 3f50d70ed3fc54..b26e248febe64b 100644 --- a/regression-test/data/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.out +++ b/regression-test/data/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !schema_false_false -- a array Yes false \N NONE -b text Yes false \N NONE +b varbinary(65533) Yes false \N NONE id int Yes false \N NONE m map Yes false \N NONE s struct Yes false \N NONE @@ -20,12 +20,12 @@ u uuid Yes false \N NONE 4 00000000-0000-0000-0000-000000000000 \N \N -- !schema_false_true -- -a array Yes false \N NONE +a array Yes false \N NONE b varbinary(65533) Yes false \N NONE id int Yes false \N NONE -m map Yes false \N NONE -s struct Yes false \N NONE -u varbinary(65533) Yes false \N NONE +m map Yes false \N NONE +s struct Yes false \N NONE +u uuid Yes false \N NONE -- !values_false_true -- 1 00112233-4455-6677-8899-aabbccddeeff ["00112233-4455-6677-8899-aabbccddeeff", null] ["00000000-0000-0000-0000-000000000000", "ffffffff-ffff-ffff-ffff-ffffffffffff"] ["80000000-0000-0000-0000-000000000000", null] 80000000-0000-0000-0000-000000000000 0000FF 6 @@ -41,7 +41,7 @@ u varbinary(65533) Yes false \N NONE -- !schema_true_false -- a array Yes false \N NONE -b text Yes false \N NONE +b varbinary(65533) Yes false \N NONE id int Yes false \N NONE m map Yes false \N NONE s struct Yes false \N NONE @@ -60,12 +60,12 @@ u uuid Yes false \N NONE 4 00000000-0000-0000-0000-000000000000 \N \N -- !schema_true_true -- -a array Yes false \N NONE +a array Yes false \N NONE b varbinary(65533) Yes false \N NONE id int Yes false \N NONE -m map Yes false \N NONE -s struct Yes false \N NONE -u varbinary(65533) Yes false \N NONE +m map Yes false \N NONE +s struct Yes false \N NONE +u uuid Yes false \N NONE -- !values_true_true -- 1 00112233-4455-6677-8899-aabbccddeeff ["00112233-4455-6677-8899-aabbccddeeff", null] ["00000000-0000-0000-0000-000000000000", "ffffffff-ffff-ffff-ffff-ffffffffffff"] ["80000000-0000-0000-0000-000000000000", null] 80000000-0000-0000-0000-000000000000 0000FF 6 diff --git a/regression-test/data/datatype_p0/uuid/test_uuid_parquet_native.out b/regression-test/data/datatype_p0/uuid/test_uuid_parquet_native.out index 7fe0885ecc2ab1..9862669c7bbe5b 100644 --- a/regression-test/data/datatype_p0/uuid/test_uuid_parquet_native.out +++ b/regression-test/data/datatype_p0/uuid/test_uuid_parquet_native.out @@ -1,23 +1,23 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !binary_schema -- id int Yes false \N NONE -u varbinary(16) Yes false \N NONE -a array Yes false \N NONE -s struct Yes false \N NONE +u text Yes false \N NONE +a array Yes false \N NONE +s struct Yes false \N NONE -- !binary_values -- 0 \N \N -1 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -2 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -3 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -4 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -5 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +1 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +2 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +3 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +4 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +5 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff 6 \N \N -7 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -8 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -9 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -10 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -11 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +7 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +8 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +9 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +10 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +11 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff -- !external_schema -- id int Yes false \N NONE @@ -111,23 +111,23 @@ ffffffff-ffff-ffff-ffff-ffffffffffff 682 1396736 -- !binary_schema -- id int Yes false \N NONE -u varbinary(16) Yes false \N NONE -a array Yes false \N NONE -s struct Yes false \N NONE +u text Yes false \N NONE +a array Yes false \N NONE +s struct Yes false \N NONE -- !binary_values -- 0 \N \N -1 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -2 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -3 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -4 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -5 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +1 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +2 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +3 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +4 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +5 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff 6 \N \N -7 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -8 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -9 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -10 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -11 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +7 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +8 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +9 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +10 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +11 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff -- !external_schema -- id int Yes false \N NONE @@ -221,23 +221,23 @@ ffffffff-ffff-ffff-ffff-ffffffffffff 682 1396736 -- !binary_schema -- id int Yes false \N NONE -u varbinary(16) Yes false \N NONE -a array Yes false \N NONE -s struct Yes false \N NONE +u text Yes false \N NONE +a array Yes false \N NONE +s struct Yes false \N NONE -- !binary_values -- 0 \N \N -1 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -2 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -3 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -4 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -5 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +1 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +2 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +3 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +4 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +5 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff 6 \N \N -7 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -8 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -9 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -10 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -11 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +7 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +8 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +9 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +10 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +11 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff -- !external_schema -- id int Yes false \N NONE @@ -331,23 +331,23 @@ ffffffff-ffff-ffff-ffff-ffffffffffff 682 1396736 -- !binary_schema -- id int Yes false \N NONE -u varbinary(16) Yes false \N NONE -a array Yes false \N NONE -s struct Yes false \N NONE +u text Yes false \N NONE +a array Yes false \N NONE +s struct Yes false \N NONE -- !binary_values -- 0 \N \N -1 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -2 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -3 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -4 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -5 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +1 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +2 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +3 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +4 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +5 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff 6 \N \N -7 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -8 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -9 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -10 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -11 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +7 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +8 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +9 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +10 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +11 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff -- !external_schema -- id int Yes false \N NONE diff --git a/regression-test/data/external_table_p0/hive/test_hive_uuid_binary.out b/regression-test/data/external_table_p0/hive/test_hive_uuid_binary.out new file mode 100644 index 00000000000000..226cdf02386d19 --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_uuid_binary.out @@ -0,0 +1,81 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !values_parquet_false_false -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_parquet_false_false -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_parquet_false_true -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_parquet_false_true -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_parquet_true_false -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_parquet_true_false -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_parquet_true_true -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_parquet_true_true -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_orc_false_false -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_orc_false_false -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_orc_false_true -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_orc_false_true -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_orc_true_false -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_orc_true_false -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_orc_true_true -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_orc_true_true -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.out new file mode 100644 index 00000000000000..8c9e82c30c8acc --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.out @@ -0,0 +1,72 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !false_UTC -- +{"ts":"1969-12-31 23:59:58.999999", "tz":"1969-12-31 23:59:58.999999+00:00"} +{"ts":"1969-12-31 23:59:59.999999", "tz":"1969-12-31 23:59:59.999999+00:00"} +{"ts":"1970-01-01 00:00:00.000001", "tz":"1970-01-01 00:00:00.000001+00:00"} +{"ts":"1970-01-01 00:00:00.000999", "tz":"1970-01-01 00:00:00.000999+00:00"} +{"ts":"1970-01-01 00:00:00.001000", "tz":"1970-01-01 00:00:00.001000+00:00"} +{"ts":"1970-01-01 00:00:01.234567", "tz":"1970-01-01 00:00:01.234567+00:00"} +{"ts":null, "tz":null} + +-- !false_UTC_ntz -- +\N +1969-12-31T23:59:58.999999 +1969-12-31T23:59:59.999999 +1970-01-01T00:00:00.000001 +1970-01-01T00:00:00.000999 +1970-01-01T00:00:00.001 +1970-01-01T00:00:01.234567 + +-- !false_Asia_Shanghai -- +{"ts":"1969-12-31 23:59:58.999999", "tz":"1970-01-01 07:59:58.999999+08:00"} +{"ts":"1969-12-31 23:59:59.999999", "tz":"1970-01-01 07:59:59.999999+08:00"} +{"ts":"1970-01-01 00:00:00.000001", "tz":"1970-01-01 08:00:00.000001+08:00"} +{"ts":"1970-01-01 00:00:00.000999", "tz":"1970-01-01 08:00:00.000999+08:00"} +{"ts":"1970-01-01 00:00:00.001000", "tz":"1970-01-01 08:00:00.001000+08:00"} +{"ts":"1970-01-01 00:00:01.234567", "tz":"1970-01-01 08:00:01.234567+08:00"} +{"ts":null, "tz":null} + +-- !false_Asia_Shanghai_ntz -- +\N +1969-12-31T23:59:58.999999 +1969-12-31T23:59:59.999999 +1970-01-01T00:00:00.000001 +1970-01-01T00:00:00.000999 +1970-01-01T00:00:00.001 +1970-01-01T00:00:01.234567 + +-- !true_UTC -- +{"ts":"1969-12-31 23:59:58.999999", "tz":"1969-12-31 23:59:58.999999+00:00"} +{"ts":"1969-12-31 23:59:59.999999", "tz":"1969-12-31 23:59:59.999999+00:00"} +{"ts":"1970-01-01 00:00:00.000001", "tz":"1970-01-01 00:00:00.000001+00:00"} +{"ts":"1970-01-01 00:00:00.000999", "tz":"1970-01-01 00:00:00.000999+00:00"} +{"ts":"1970-01-01 00:00:00.001000", "tz":"1970-01-01 00:00:00.001000+00:00"} +{"ts":"1970-01-01 00:00:01.234567", "tz":"1970-01-01 00:00:01.234567+00:00"} +{"ts":null, "tz":null} + +-- !true_UTC_ntz -- +\N +1969-12-31T23:59:58.999999 +1969-12-31T23:59:59.999999 +1970-01-01T00:00:00.000001 +1970-01-01T00:00:00.000999 +1970-01-01T00:00:00.001 +1970-01-01T00:00:01.234567 + +-- !true_Asia_Shanghai -- +{"ts":"1969-12-31 23:59:58.999999", "tz":"1970-01-01 07:59:58.999999+08:00"} +{"ts":"1969-12-31 23:59:59.999999", "tz":"1970-01-01 07:59:59.999999+08:00"} +{"ts":"1970-01-01 00:00:00.000001", "tz":"1970-01-01 08:00:00.000001+08:00"} +{"ts":"1970-01-01 00:00:00.000999", "tz":"1970-01-01 08:00:00.000999+08:00"} +{"ts":"1970-01-01 00:00:00.001000", "tz":"1970-01-01 08:00:00.001000+08:00"} +{"ts":"1970-01-01 00:00:01.234567", "tz":"1970-01-01 08:00:01.234567+08:00"} +{"ts":null, "tz":null} + +-- !true_Asia_Shanghai_ntz -- +\N +1969-12-31T23:59:58.999999 +1969-12-31T23:59:59.999999 +1970-01-01T00:00:00.000001 +1970-01-01T00:00:00.000999 +1970-01-01T00:00:00.001 +1970-01-01T00:00:01.234567 diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out index 0094b0f105f0c5..88a21fa45d5c5c 100644 --- a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out @@ -7,7 +7,7 @@ true 11 111 1.1 1.1 1111 1234.5678 1234.567890 123456789012345678.123456789012 a 0 PARQUET 2 {1:2, 2:2, 3:2, 4:2, 5:2, 6:2, 7:2, 8:2, 9:2, 10:2, 11:2, 12:2} {1:0, 2:0, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0, 10:0, 11:0, 12:0} {1:0x00, 2:0x0B000000, 3:0x6F00000000000000, 4:0xCDCC8C3F, 5:0x9A9999999999F13F, 6:0x00000457, 7:0x00BC614E, 8:0x00000000499602D2, 9:0x000000018EE90FF6C373E0393713FA14, 10:0x616161, 11:0x9E4B0000, 12:0x005CE70F33F10500} {1:0x01, 2:0x16000000, 3:0xDE00000000000000, 4:0xCDCC0C40, 5:0x9A99999999990140, 6:0x000008AE, 7:0x05397FB1, 8:0x000000020A75E124, 9:0x0000000C7748819DFFB62505316873CB, 10:0x626262, 11:0x434C0000, 12:0x40D91FA833FE0500} -- !sql_2 -- -{"bigint_col":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":111, "upper_bound":222}, "boolean_col":{"column_size":33, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":0, "upper_bound":1}, "date_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"2023-01-01", "upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"2023-01-01 20:34:56.000000", "upper_bound":"2023-06-16 07:45:01.000000"}, "decimal_col1":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1111, "upper_bound":2222}, "decimal_col2":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1234.5678, "upper_bound":8765.4321}, "decimal_col3":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1234.567890, "upper_bound":8765.432100}, "decimal_col4":{"column_size":90, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":123456789012345678.123456789012, "upper_bound":987654321098765432.987654321099}, "double_col":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "float_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "int_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":11, "upper_bound":22}, "string_col":{"column_size":72, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"aaa", "upper_bound":"bbb"}} +{"bigint_col":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":111, "upper_bound":222}, "boolean_col":{"column_size":33, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":0, "upper_bound":1}, "date_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"2023-01-01", "upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"2023-01-01 12:34:56.000000", "upper_bound":"2023-06-15 23:45:01.000000"}, "decimal_col1":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1111, "upper_bound":2222}, "decimal_col2":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1234.5678, "upper_bound":8765.4321}, "decimal_col3":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1234.567890, "upper_bound":8765.432100}, "decimal_col4":{"column_size":90, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":123456789012345678.123456789012, "upper_bound":987654321098765432.987654321099}, "double_col":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "float_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "int_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":11, "upper_bound":22}, "string_col":{"column_size":72, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"aaa", "upper_bound":"bbb"}} -- !sql_3 -- false 22 222 2.2 2.2 2222 8765.4321 8765.432100 987654321098765432.987654321099 bbb 2023-06-15 2023-06-15T23:45:01 @@ -17,7 +17,7 @@ true 11 111 1.1 1.1 1111 1234.5678 1234.567890 123456789012345678.123456789012 a 0 ORC 2 {1:2, 2:2, 3:2, 4:2, 5:2, 6:2, 7:2, 8:2, 9:2, 10:2, 11:2, 12:2} {} {1:0x00, 2:0x0B00000000000000, 3:0x6F00000000000000, 4:0xCDCC8C3F, 5:0x9A9999999999F13F, 6:0x0457, 7:0x00BC614E, 8:0x075BCD15, 9:0x018EE90FF6C373E0393713FA14, 10:0x616161, 11:0x9E4B0000, 12:0x005CE70F33F10500} {1:0x01, 2:0x1600000000000000, 3:0xDE00000000000000, 4:0xCDCC0C40, 5:0x9A99999999990140, 6:0x08AE, 7:0x05397FB1, 8:0x05397FB1, 9:0x0C7748819DFFB62505316873CB, 10:0x626262, 11:0x434C0000, 12:0x40D91FA833FE0500} -- !sql_5 -- -{"bigint_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":111, "upper_bound":222}, "boolean_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":0, "upper_bound":1}, "date_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":"2023-01-01", "upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":"2023-01-01 20:34:56.000000", "upper_bound":"2023-06-16 07:45:01.000000"}, "decimal_col1":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1111, "upper_bound":2222}, "decimal_col2":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1234.5678, "upper_bound":8765.4321}, "decimal_col3":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":123.456789, "upper_bound":87.654321}, "decimal_col4":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":123456789012345678.123456789012, "upper_bound":987654321098765432.987654321099}, "double_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "float_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "int_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":11, "upper_bound":22}, "string_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":"aaa", "upper_bound":"bbb"}} +{"bigint_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":111, "upper_bound":222}, "boolean_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":0, "upper_bound":1}, "date_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":"2023-01-01", "upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":"2023-01-01 12:34:56.000000", "upper_bound":"2023-06-15 23:45:01.000000"}, "decimal_col1":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1111, "upper_bound":2222}, "decimal_col2":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1234.5678, "upper_bound":8765.4321}, "decimal_col3":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":123.456789, "upper_bound":87.654321}, "decimal_col4":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":123456789012345678.123456789012, "upper_bound":987654321098765432.987654321099}, "double_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "float_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "int_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":11, "upper_bound":22}, "string_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":"aaa", "upper_bound":"bbb"}} -- !sql_6 -- {1:0x00, 2:0x0B000000, 3:0x6F00000000000000, 4:0xCDCC8C3F, 5:0x9A9999999999F13F, 6:0x00000457, 7:0x00BC614E, 8:0x00000000499602D2, 9:0x000000018EE90FF6C373E0393713FA14, 10:0x616161, 11:0x9E4B0000, 12:0x005CE70F33F10500} diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_catalog_varbinary.out b/regression-test/data/external_table_p0/paimon/test_paimon_catalog_varbinary.out index e65f32a2ad6113..09b85d61c88549 100644 Binary files a/regression-test/data/external_table_p0/paimon/test_paimon_catalog_varbinary.out and b/regression-test/data/external_table_p0/paimon/test_paimon_catalog_varbinary.out differ diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out index cb2f86d7ba47e4..c3b6fac5935862 100644 Binary files a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out and b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out differ diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group5_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group5_orc_files.out index 3440963fcaff40..9052566df19922 100644 --- a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group5_orc_files.out +++ b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group5_orc_files.out @@ -4,9 +4,9 @@ 456 2 -- !test_2 -- -[{"elem1":1, "elem2":"123", "elem3":9.8, "nested":[{"elem1":1, "elem2":"123", "elem3":9.8}, {"elem1":2, "elem2":"456", "elem3":10.12}, {"elem1":3, "elem2":"789", "elem3":11.14}]}, {"elem1":2, "elem2":"456", "elem3":10.12, "nested":[{"elem1":4, "elem2":"101112", "elem3":123.8}, {"elem1":5, "elem2":"131415", "elem3":10.2}, {"elem1":6, "elem2":"161718", "elem3":11.414}]}, {"elem1":3, "elem2":"789", "elem3":11.14, "nested":[{"elem1":7, "elem2":"101", "elem3":13.08}, {"elem1":8, "elem2":"415", "elem3":1.12}, {"elem1":9, "elem2":"118", "elem3":0.414}]}] -[{"elem1":4, "elem2":"101112", "elem3":123.8, "nested":[{"elem1":1, "elem2":"123", "elem3":9.8}, {"elem1":2, "elem2":"456", "elem3":10.12}, {"elem1":3, "elem2":"789", "elem3":11.14}]}, {"elem1":5, "elem2":"131415", "elem3":10.2, "nested":[{"elem1":4, "elem2":"101112", "elem3":123.8}, {"elem1":5, "elem2":"131415", "elem3":10.2}, {"elem1":6, "elem2":"161718", "elem3":11.414}]}, {"elem1":6, "elem2":"161718", "elem3":11.414, "nested":[{"elem1":7, "elem2":"101", "elem3":13.08}, {"elem1":8, "elem2":"415", "elem3":1.12}, {"elem1":9, "elem2":"118", "elem3":0.414}]}] -[{"elem1":7, "elem2":"101", "elem3":13.08, "nested":[{"elem1":1, "elem2":"123", "elem3":9.8}, {"elem1":2, "elem2":"456", "elem3":10.12}, {"elem1":3, "elem2":"789", "elem3":11.14}]}, {"elem1":8, "elem2":"415", "elem3":1.12, "nested":[{"elem1":4, "elem2":"101112", "elem3":123.8}, {"elem1":5, "elem2":"131415", "elem3":10.2}, {"elem1":6, "elem2":"161718", "elem3":11.414}]}, {"elem1":9, "elem2":"118", "elem3":0.414, "nested":[{"elem1":7, "elem2":"101", "elem3":13.08}, {"elem1":8, "elem2":"415", "elem3":1.12}, {"elem1":9, "elem2":"118", "elem3":0.414}]}] +[{"elem1":1, "elem2":0x313233, "elem3":9.8, "nested":[{"elem1":1, "elem2":0x313233, "elem3":9.8}, {"elem1":2, "elem2":0x343536, "elem3":10.12}, {"elem1":3, "elem2":0x373839, "elem3":11.14}]}, {"elem1":2, "elem2":0x343536, "elem3":10.12, "nested":[{"elem1":4, "elem2":0x313031313132, "elem3":123.8}, {"elem1":5, "elem2":0x313331343135, "elem3":10.2}, {"elem1":6, "elem2":0x313631373138, "elem3":11.414}]}, {"elem1":3, "elem2":0x373839, "elem3":11.14, "nested":[{"elem1":7, "elem2":0x313031, "elem3":13.08}, {"elem1":8, "elem2":0x343135, "elem3":1.12}, {"elem1":9, "elem2":0x313138, "elem3":0.414}]}] +[{"elem1":4, "elem2":0x313031313132, "elem3":123.8, "nested":[{"elem1":1, "elem2":0x313233, "elem3":9.8}, {"elem1":2, "elem2":0x343536, "elem3":10.12}, {"elem1":3, "elem2":0x373839, "elem3":11.14}]}, {"elem1":5, "elem2":0x313331343135, "elem3":10.2, "nested":[{"elem1":4, "elem2":0x313031313132, "elem3":123.8}, {"elem1":5, "elem2":0x313331343135, "elem3":10.2}, {"elem1":6, "elem2":0x313631373138, "elem3":11.414}]}, {"elem1":6, "elem2":0x313631373138, "elem3":11.414, "nested":[{"elem1":7, "elem2":0x313031, "elem3":13.08}, {"elem1":8, "elem2":0x343135, "elem3":1.12}, {"elem1":9, "elem2":0x313138, "elem3":0.414}]}] +[{"elem1":7, "elem2":0x313031, "elem3":13.08, "nested":[{"elem1":1, "elem2":0x313233, "elem3":9.8}, {"elem1":2, "elem2":0x343536, "elem3":10.12}, {"elem1":3, "elem2":0x373839, "elem3":11.14}]}, {"elem1":8, "elem2":0x343135, "elem3":1.12, "nested":[{"elem1":4, "elem2":0x313031313132, "elem3":123.8}, {"elem1":5, "elem2":0x313331343135, "elem3":10.2}, {"elem1":6, "elem2":0x313631373138, "elem3":11.414}]}, {"elem1":9, "elem2":0x313138, "elem3":0.414, "nested":[{"elem1":7, "elem2":0x313031, "elem3":13.08}, {"elem1":8, "elem2":0x343135, "elem3":1.12}, {"elem1":9, "elem2":0x313138, "elem3":0.414}]}] -- !test_3 -- 1 2024-06-30T20:00 @@ -16,9 +16,9 @@ 456 2 -- !test_5 -- -[{"elem1":1, "elem2":"123", "elem3":9.8}, {"elem1":2, "elem2":"456", "elem3":10.12}, {"elem1":3, "elem2":"789", "elem3":11.14}] -[{"elem1":4, "elem2":"101112", "elem3":123.8}, {"elem1":5, "elem2":"131415", "elem3":10.2}, {"elem1":6, "elem2":"161718", "elem3":11.414}] -[{"elem1":7, "elem2":"101", "elem3":13.08}, {"elem1":8, "elem2":"415", "elem3":1.12}, {"elem1":9, "elem2":"118", "elem3":0.414}] +[{"elem1":1, "elem2":0x313233, "elem3":9.8}, {"elem1":2, "elem2":0x343536, "elem3":10.12}, {"elem1":3, "elem2":0x373839, "elem3":11.14}] +[{"elem1":4, "elem2":0x313031313132, "elem3":123.8}, {"elem1":5, "elem2":0x313331343135, "elem3":10.2}, {"elem1":6, "elem2":0x313631373138, "elem3":11.414}] +[{"elem1":7, "elem2":0x313031, "elem3":13.08}, {"elem1":8, "elem2":0x343135, "elem3":1.12}, {"elem1":9, "elem2":0x313138, "elem3":0.414}] -- !test_6 -- \N \N diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out index f31f2a50d2fba9..d73f0dce7558e7 100644 Binary files a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out and b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out differ diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group7_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group7_orc_files.out index bf0d7b8a5151cf..1150886460a43b 100644 Binary files a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group7_orc_files.out and b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group7_orc_files.out differ diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group0.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group0.out index cdb8aa66f70d12..632334c5d06e8e 100644 Binary files a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group0.out and b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group0.out differ diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out index 899acd4edd5f0c..2a8a902028d631 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out @@ -195,12 +195,12 @@ false 1 2 3 10 1.2 val_1 val_1 HEARTS false 1 2 3 10 1.2 val_1 val_1 HEARTS ["ar false 3 4 5 30 3.2 val_3 val_3 CLUBS \N \N \N \N \N \N \N \N \N ["arr_3", "arr_4", "arr_5"] [3] {3:"val_3", 4:"val_4", 5:"val_5"} {3:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}], 4:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}], 5:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}]} false 5 6 7 50 5.2 val_5 val_5 HEARTS false 5 6 7 50 5.2 val_5 val_5 HEARTS ["arr_5", "arr_6", "arr_7"] [5] {5:"val_5", 6:"val_6", 7:"val_7"} {5:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}], 6:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}], 7:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}]} false 7 8 9 70 7.2 val_7 val_7 CLUBS false 7 8 9 70 7.2 val_7 val_7 CLUBS ["arr_7", "arr_8", "arr_9"] [7] {7:"val_7", 8:"val_8", 9:"val_9"} {7:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}], 8:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}], 9:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}]} -false 9 10 11 90 9.199999999999999 val_9 val_9 HEARTS \N \N \N \N \N \N \N \N \N ["arr_9", "arr_10", "arr_11"] [9] {9:"val_9", 10:"val_10", 11:"val_11"} {9:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 10:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 11:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}]} +false 9 10 11 90 9.2 val_9 val_9 HEARTS \N \N \N \N \N \N \N \N \N ["arr_9", "arr_10", "arr_11"] [9] {9:"val_9", 10:"val_10", 11:"val_11"} {9:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 10:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 11:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}]} true 0 1 2 0 0.2 val_0 val_0 SPADES \N \N \N \N \N \N \N \N \N ["arr_0", "arr_1", "arr_2"] [0] {0:"val_0", 1:"val_1", 2:"val_2"} {0:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}], 1:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}], 2:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}]} true 2 3 4 20 2.2 val_2 val_2 DIAMONDS true 2 3 4 20 2.2 val_2 val_2 DIAMONDS ["arr_2", "arr_3", "arr_4"] [2] {2:"val_2", 3:"val_3", 4:"val_4"} {2:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}], 3:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}], 4:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}]} true 4 5 6 40 4.2 val_4 val_4 SPADES true 4 5 6 40 4.2 val_4 val_4 SPADES ["arr_4", "arr_5", "arr_6"] [4] {4:"val_4", 5:"val_5", 6:"val_6"} {4:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}], 5:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}], 6:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}]} true 6 7 8 60 6.2 val_6 val_6 DIAMONDS \N \N \N \N \N \N \N \N \N ["arr_6", "arr_7", "arr_8"] [6] {6:"val_6", 7:"val_7", 8:"val_8"} {6:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}], 7:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}], 8:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}]} -true 8 9 10 80 8.199999999999999 val_8 val_8 SPADES true 8 9 10 80 8.199999999999999 val_8 val_8 SPADES ["arr_8", "arr_9", "arr_10"] [8] {8:"val_8", 9:"val_9", 10:"val_10"} {8:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 9:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 10:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}]} +true 8 9 10 80 8.2 val_8 val_8 SPADES true 8 9 10 80 8.2 val_8 val_8 SPADES ["arr_8", "arr_9", "arr_10"] [8] {8:"val_8", 9:"val_9", 10:"val_10"} {8:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 9:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 10:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}]} -- !test_25 -- {"duration":"111222333444"} diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group3.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group3.out index 93c2fd8c672e39..b079c6f3de820d 100644 Binary files a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group3.out and b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group3.out differ diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group4.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group4.out index 231277cdb63189..56c373b4510691 100644 Binary files a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group4.out and b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group4.out differ diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out index 2b47e7d4575b97..a6f0aae03f9d29 100644 Binary files a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out and b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out differ diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out index b50d2edb840fa5..a31770af13554e 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out @@ -166,7 +166,7 @@ value 0 value value 0 value value value value value value value value value valu 20230110191854854 20230110191854854_0_2 AA0 0df0196b-f46f-43f5-8cf0-06fad7143af3-0_0-27-35_20230110191854854.parquet AA0 10 0 hello [{"array":[10, 20, 30]}, {"array":[40, 50, 60, 70]}] {"key1":[1, 10], "key2":[2, 20], "key3":null} {"a":[10, 20], "b":{"key1":10}, "c":{"a":[10, 20], "b":{"a":10, "b":"world"}}} -- !test_17 -- -6  +6 0x0F -- !test_18 -- 2002-09-08 2003-01-28T05:11:07.683 8X qp3jpOhuc3eeZn03znr false 4 -900 64505843 8210173644705794 3.94371648E8 103534.6075051678 -900.39 -186247140.75010 114045.570460308 @@ -239,16 +239,16 @@ value 0 value value 0 value value value value value value value value value valu {"0":0, "1":1, "2":2, "3":3, "4":4} -- !test_25 -- -2 20190519150339 1558249419649 1.0.1_pre 045e52cffa420a3a 180.164.69.154 \N Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_2 like Mac OS X; zh-CN) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/15C202 UCBrowser/11.7.7.1031 Mobile AliApp(TUnionSDK/0.1.20) 1242*2208 http://m.sohu.com/a/305789055_120103662 https://so.m.sm.cn/s?q=%E6%9D%83%E5%8A%9B%E7%9A%84%E6%B8%B8%E6%88%8F%E6%B4%9B%E5%85%8B&uc_param_str=dnntnwvepffrgibijbprsvdsei&dn=18341617082-ee4b21c8&nt=2&nw=WIFI&ve=11.7.7.1031&pf=44&fr=iphone&bi=99 \N 3 1 \N \N smwp.content.0.0.1558249419151vih2xDD \N \N -2 20190519150339 1558249419651 1.0.1_pre 042b08b961ca09a4 211.161.73.18 \N Mozilla/5.0 (iPhone; CPU iPhone OS 11_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.0 Mobile/15E148 Safari/604.1 750*1334 http://m.sohu.com/a/249765423_100089662 https://m.baidu.com/from=844b/bd_page_type=1/ssid=0/uid=0/pu=usm%401%2Csz%401320_2001%2Cta%40iphone_1_11.4_25_11.0/baiduid=2BA3A9AA740E480F7EFA68288AC2C4DD/w=0_10_/t=iphone/l=1/tc?clk_type=1&vit=osres \N 3 1 \N \N smwp.content.0.0.1558249419547H7q8WCH \N \N -2 20190519150339 1558249419678 1.0.1_pre 1905061742091531 117.136.86.175 \N Mozilla/5.0 (Linux; U; Android 8.1.0; zh-cn; Redmi 5A Build/OPM1.171019.026) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/9.2 Mobile Safari/537.36 720*1280 http://m.sohu.com/cm/313048470_100274493?spm=smwp.content.comment-reply.3.1558248672423N82BIA3 http://m.sohu.com/a/313048470_100274493?scm=1002.b000b.fa0137014e0157.ARTICLE_REC&spm=smwp.content.fd-d.1.1558247629506v8ptViu \N 5 1 \N \N smwp.comment.comment.44.1558249289410YENLtLo smwp.content.comment-reply.3.1558248672423N82BIA3 \N -2 20190519150339 1558249419678 1.0.1_pre 1905061742091531 117.136.86.175 \N Mozilla/5.0 (Linux; U; Android 8.1.0; zh-cn; Redmi 5A Build/OPM1.171019.026) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/9.2 Mobile Safari/537.36 720*1280 http://m.sohu.com/cm/313048470_100274493?spm=smwp.content.comment-reply.3.1558248672423N82BIA3 http://m.sohu.com/a/313048470_100274493?scm=1002.b000b.fa0137014e0157.ARTICLE_REC&spm=smwp.content.fd-d.1.1558247629506v8ptViu \N 5 1 \N \N smwp.comment.comment.45.1558249289410YENLtLo smwp.content.comment-reply.3.1558248672423N82BIA3 \N -2 20190519150339 1558249419683 1.0.1_pre 1901221157475PE2 42.53.118.200 \N Mozilla/5.0 (Linux; U; Android 9; zh-CN; HMA-AL00 Build/HUAWEIHMA-AL00) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.108 UCBrowser/12.4.4.1024 Mobile Safari/537.36 1080*2244 https://m.sohu.com/subject/315779?scm=undefined&spm=smwp.fb-xj-home.fd-d.2.1558249363211CkASa5u&_trans_=000012_uc_wzdhx https://m.sohu.com/z/laliga?spm=smwp.fb-sports-home.navs.8.1558249359405AX3RCDd&_trans_=000012_uc_wzdhx \N 8 1 314962078 461392 smwp.subject.column-1.5.1558249373129156jnrb smwp.fb-xj-home.fd-d.2.1558249363211CkASa5u \N -2 20190519150339 1558249419696 1.0.1_pre 045e52cde3500800 117.136.86.30 \N Mozilla/5.0 (Linux; U; Android 8.0zh-cn; ONEPLUS A6010 Build/PKQ1.180716.001) AppleWebKit/537.36 (KHTML, like Gecko)Version/4.0 Chrome/57.0.2987.132 MQQBrowser/8.1 Mobile Safari/537.36 1080*2341 https://m.sohu.com/a/211545780_99914238/?pvid=000115_3w_a https://m.baidu.com/from=1000953f/bd_page_type=1/ssid=0/uid=0/pu=usm%402%2Csz%40320_1002%2Cta%40iphone_2_8.0_2_12137.1/baiduid=121BF90FD6F83CBAB09EF3561492B629/w=0_10_/t=iphone/l=1/tc?clk_type=1&vit=o \N 3 1 \N \N smwp.content.fx.2.1558249418813CreAoGd \N \N -2 20190519150339 1558249419701 1.0.1_pre 190304212717ZVOJ 27.214.184.205 \N Mozilla/5.0 (Linux; Android 8.0.0; SM-G9350 Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 Mobile Safari/537.36;haowangzhi;ebrowser 2230 1080*1920 http://m.sohu.com/limit \N \N 99 1 314721278 821681 smwp.home.fd-auto.4.1558248949655lHmQ7dl \N \N -2 20190519150339 1558249419701 1.0.1_pre 190304212717ZVOJ 27.214.184.205 \N Mozilla/5.0 (Linux; Android 8.0.0; SM-G9350 Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 Mobile Safari/537.36;haowangzhi;ebrowser 2230 1080*1920 http://m.sohu.com/limit \N \N 99 1 314909226 524232 smwp.home.fd-auto.6.1558248949655lHmQ7dl \N \N -2 20190519150339 1558249419701 1.0.1_pre 190304212717ZVOJ 27.214.184.205 \N Mozilla/5.0 (Linux; Android 8.0.0; SM-G9350 Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 Mobile Safari/537.36;haowangzhi;ebrowser 2230 1080*1920 http://m.sohu.com/limit \N \N 99 1 314909730 120039 smwp.home.fd-auto.5.1558248949655lHmQ7dl \N \N -2 20190519150339 1558249419702 1.0.1_pre 045e52be29760ee7 218.26.54.70 \N Mozilla/5.0 (Linux; U; Android 8.0.0; zh-cn; SM-G9500 Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/8.7 Mobile Safari/537.36 1080*2220 http://m.sohu.com/a/313321927_120123535 https://m.sogou.com/web/searchList.jsp?pid=sogou-waps-bb576d1513ba8ee9&keyword=%E8%BD%A6%E6%8D%A2%E6%8C%A1%E9%A3%8E%E7%8E%BB%E7%92%83%E8%83%BD%E4%B9%B0%E4%B8%8D \N 3 1 \N \N smwp.content.fx.1.1558249398059Unr8hGe \N \N +0x32 20190519150339 1558249419649 0x312E302E315F707265 0x30343565353263666661343230613361 0x3138302E3136342E36392E313534 \N 0x4D6F7A696C6C612F352E3020286950686F6E653B20435055206950686F6E65204F532031315F325F32206C696B65204D6163204F5320583B207A682D434E29204170706C655765624B69742F3533372E35312E3120284B48544D4C2C206C696B65204765636B6F29204D6F62696C652F31354332303220554342726F777365722F31312E372E372E31303331204D6F62696C652020416C694170702854556E696F6E53444B2F302E312E323029 0x313234322A32323038 0x687474703A2F2F6D2E736F68752E636F6D2F612F3330353738393035355F313230313033363632 0x68747470733A2F2F736F2E6D2E736D2E636E2F733F713D2545362539442538332545352538412539422545372539412538342545362542382542382545362538382538462545362542342539422545352538352538422675635F706172616D5F7374723D646E6E746E77766570666672676962696A62707273766473656926646E3D31383334313631373038322D6565346232316338266E743D32266E773D574946492676653D31312E372E372E313033312670663D34342666723D6970686F6E652662693D3939 \N 0x33 0x31 \N \N 0x736D77702E636F6E74656E742E302E302E3135353832343934313931353176696832784444 \N \N +0x32 20190519150339 1558249419651 0x312E302E315F707265 0x30343262303862393631636130396134 0x3231312E3136312E37332E3138 \N 0x4D6F7A696C6C612F352E3020286950686F6E653B20435055206950686F6E65204F532031315F34206C696B65204D6163204F53205829204170706C655765624B69742F3630352E312E313520284B48544D4C2C206C696B65204765636B6F292056657273696F6E2F31312E30204D6F62696C652F313545313438205361666172692F3630342E31 0x3735302A31333334 0x687474703A2F2F6D2E736F68752E636F6D2F612F3234393736353432335F313030303839363632 0x68747470733A2F2F6D2E62616964752E636F6D2F66726F6D3D383434622F62645F706167655F747970653D312F737369643D302F7569643D302F70753D75736D25343031253243737A253430313332305F3230303125324374612534306970686F6E655F315F31312E345F32355F31312E302F626169647569643D32424133413941413734304534383046374546413638323838414332433444442F773D305F31305F2F743D6970686F6E652F6C3D312F74633F636C6B5F747970653D31267669743D6F73726573 \N 0x33 0x31 \N \N 0x736D77702E636F6E74656E742E302E302E3135353832343934313935343748377138574348 \N \N +0x32 20190519150339 1558249419678 0x312E302E315F707265 0x31393035303631373432303931353331 0x3131372E3133362E38362E313735 \N 0x4D6F7A696C6C612F352E3020284C696E75783B20553B20416E64726F696420382E312E303B207A682D636E3B205265646D69203541204275696C642F4F504D312E3137313031392E30323629204170706C655765624B69742F3533372E333620284B48544D4C2C206C696B65204765636B6F292056657273696F6E2F342E30204368726F6D652F35372E302E323938372E313332204D515142726F777365722F392E32204D6F62696C65205361666172692F3533372E3336 0x3732302A31323830 0x687474703A2F2F6D2E736F68752E636F6D2F636D2F3331333034383437305F3130303237343439333F73706D3D736D77702E636F6E74656E742E636F6D6D656E742D7265706C792E332E313535383234383637323432334E383242494133 0x687474703A2F2F6D2E736F68752E636F6D2F612F3331333034383437305F3130303237343439333F73636D3D313030322E62303030622E66613031333730313465303135372E41525449434C455F5245432673706D3D736D77702E636F6E74656E742E66642D642E312E3135353832343736323935303676387074566975 \N 0x35 0x31 \N \N 0x736D77702E636F6D6D656E742E636F6D6D656E742E34342E3135353832343932383934313059454E4C744C6F 0x736D77702E636F6E74656E742E636F6D6D656E742D7265706C792E332E313535383234383637323432334E383242494133 \N +0x32 20190519150339 1558249419678 0x312E302E315F707265 0x31393035303631373432303931353331 0x3131372E3133362E38362E313735 \N 0x4D6F7A696C6C612F352E3020284C696E75783B20553B20416E64726F696420382E312E303B207A682D636E3B205265646D69203541204275696C642F4F504D312E3137313031392E30323629204170706C655765624B69742F3533372E333620284B48544D4C2C206C696B65204765636B6F292056657273696F6E2F342E30204368726F6D652F35372E302E323938372E313332204D515142726F777365722F392E32204D6F62696C65205361666172692F3533372E3336 0x3732302A31323830 0x687474703A2F2F6D2E736F68752E636F6D2F636D2F3331333034383437305F3130303237343439333F73706D3D736D77702E636F6E74656E742E636F6D6D656E742D7265706C792E332E313535383234383637323432334E383242494133 0x687474703A2F2F6D2E736F68752E636F6D2F612F3331333034383437305F3130303237343439333F73636D3D313030322E62303030622E66613031333730313465303135372E41525449434C455F5245432673706D3D736D77702E636F6E74656E742E66642D642E312E3135353832343736323935303676387074566975 \N 0x35 0x31 \N \N 0x736D77702E636F6D6D656E742E636F6D6D656E742E34352E3135353832343932383934313059454E4C744C6F 0x736D77702E636F6E74656E742E636F6D6D656E742D7265706C792E332E313535383234383637323432334E383242494133 \N +0x32 20190519150339 1558249419683 0x312E302E315F707265 0x31393031323231313537343735504532 0x34322E35332E3131382E323030 \N 0x4D6F7A696C6C612F352E3020284C696E75783B20553B20416E64726F696420393B207A682D434E3B20484D412D414C3030204275696C642F485541574549484D412D414C303029204170706C655765624B69742F3533372E333620284B48544D4C2C206C696B65204765636B6F292056657273696F6E2F342E30204368726F6D652F35372E302E323938372E31303820554342726F777365722F31322E342E342E31303234204D6F62696C65205361666172692F3533372E3336 0x313038302A32323434 0x68747470733A2F2F6D2E736F68752E636F6D2F7375626A6563742F3331353737393F73636D3D756E646566696E65642673706D3D736D77702E66622D786A2D686F6D652E66642D642E322E31353538323439333633323131436B4153613575265F7472616E735F3D3030303031325F75635F777A646878 0x68747470733A2F2F6D2E736F68752E636F6D2F7A2F6C616C6967613F73706D3D736D77702E66622D73706F7274732D686F6D652E6E6176732E382E3135353832343933353934303541583352434464265F7472616E735F3D3030303031325F75635F777A646878 \N 0x38 0x31 0x333134393632303738 0x343631333932 0x736D77702E7375626A6563742E636F6C756D6E2D312E352E313535383234393337333132393135366A6E7262 0x736D77702E66622D786A2D686F6D652E66642D642E322E31353538323439333633323131436B4153613575 \N +0x32 20190519150339 1558249419696 0x312E302E315F707265 0x30343565353263646533353030383030 0x3131372E3133362E38362E3330 \N 0x4D6F7A696C6C612F352E3020284C696E75783B20553B20416E64726F696420382E307A682D636E3B204F4E45504C5553204136303130204275696C642F504B51312E3138303731362E30303129204170706C655765624B69742F3533372E333620284B48544D4C2C206C696B65204765636B6F2956657273696F6E2F342E30204368726F6D652F35372E302E323938372E313332204D515142726F777365722F382E31204D6F62696C65205361666172692F3533372E3336 0x313038302A32333431 0x68747470733A2F2F6D2E736F68752E636F6D2F612F3231313534353738305F39393931343233382F3F707669643D3030303131355F33775F61 0x68747470733A2F2F6D2E62616964752E636F6D2F66726F6D3D31303030393533662F62645F706167655F747970653D312F737369643D302F7569643D302F70753D75736D25343032253243737A2534303332305F3130303225324374612534306970686F6E655F325F382E305F325F31323133372E312F626169647569643D31323142463930464436463833434241423039454633353631343932423632392F773D305F31305F2F743D6970686F6E652F6C3D312F74633F636C6B5F747970653D31267669743D6F \N 0x33 0x31 \N \N 0x736D77702E636F6E74656E742E66782E322E31353538323439343138383133437265416F4764 \N \N +0x32 20190519150339 1558249419701 0x312E302E315F707265 0x3139303330343231323731375A564F4A 0x32372E3231342E3138342E323035 \N 0x4D6F7A696C6C612F352E3020284C696E75783B20416E64726F696420382E302E303B20534D2D4739333530204275696C642F5231364E573B20777629204170706C655765624B69742F3533372E333620284B48544D4C2C206C696B65204765636B6F292056657273696F6E2F342E30204368726F6D652F36362E302E333335392E313236204D6F62696C65205361666172692F3533372E33363B68616F77616E677A68693B6562726F777365722032323330 0x313038302A31393230 0x687474703A2F2F6D2E736F68752E636F6D2F6C696D6974 \N \N 0x3939 0x31 0x333134373231323738 0x383231363831 0x736D77702E686F6D652E66642D6175746F2E342E313535383234383934393635356C486D5137646C \N \N +0x32 20190519150339 1558249419701 0x312E302E315F707265 0x3139303330343231323731375A564F4A 0x32372E3231342E3138342E323035 \N 0x4D6F7A696C6C612F352E3020284C696E75783B20416E64726F696420382E302E303B20534D2D4739333530204275696C642F5231364E573B20777629204170706C655765624B69742F3533372E333620284B48544D4C2C206C696B65204765636B6F292056657273696F6E2F342E30204368726F6D652F36362E302E333335392E313236204D6F62696C65205361666172692F3533372E33363B68616F77616E677A68693B6562726F777365722032323330 0x313038302A31393230 0x687474703A2F2F6D2E736F68752E636F6D2F6C696D6974 \N \N 0x3939 0x31 0x333134393039323236 0x353234323332 0x736D77702E686F6D652E66642D6175746F2E362E313535383234383934393635356C486D5137646C \N \N +0x32 20190519150339 1558249419701 0x312E302E315F707265 0x3139303330343231323731375A564F4A 0x32372E3231342E3138342E323035 \N 0x4D6F7A696C6C612F352E3020284C696E75783B20416E64726F696420382E302E303B20534D2D4739333530204275696C642F5231364E573B20777629204170706C655765624B69742F3533372E333620284B48544D4C2C206C696B65204765636B6F292056657273696F6E2F342E30204368726F6D652F36362E302E333335392E313236204D6F62696C65205361666172692F3533372E33363B68616F77616E677A68693B6562726F777365722032323330 0x313038302A31393230 0x687474703A2F2F6D2E736F68752E636F6D2F6C696D6974 \N \N 0x3939 0x31 0x333134393039373330 0x313230303339 0x736D77702E686F6D652E66642D6175746F2E352E313535383234383934393635356C486D5137646C \N \N +0x32 20190519150339 1558249419702 0x312E302E315F707265 0x30343565353262653239373630656537 0x3231382E32362E35342E3730 \N 0x4D6F7A696C6C612F352E3020284C696E75783B20553B20416E64726F696420382E302E303B207A682D636E3B20534D2D4739353030204275696C642F5231364E5729204170706C655765624B69742F3533372E333620284B48544D4C2C206C696B65204765636B6F292056657273696F6E2F342E30204368726F6D652F35372E302E323938372E313332204D515142726F777365722F382E37204D6F62696C65205361666172692F3533372E3336 0x313038302A32323230 0x687474703A2F2F6D2E736F68752E636F6D2F612F3331333332313932375F313230313233353335 0x68747470733A2F2F6D2E736F676F752E636F6D2F7765622F7365617263684C6973742E6A73703F7069643D736F676F752D776170732D62623537366431353133626138656539266B6579776F72643D254538254244254136254536253844254132254536253843254131254539254133253845254537253845254242254537253932253833254538253833254244254534254239254230254534254238253844 \N 0x33 0x31 \N \N 0x736D77702E636F6E74656E742E66782E312E31353538323439333938303539556E7238684765 \N \N -- !test_27 -- [["1"], ["2"], ["3"]] @@ -657,10 +657,10 @@ hello {"x":"world", "y":"danny"} 5 -- !test_74 -- \N \N \N \N \N -string_smith enum_smith 6.7 {"name": "smith"} {"name": "smith"} -string_smith enum_smith 6.7 {"name": "smith"} {"name": "smith"} -string_smith enum_smith 6.7 {"name": "smith"} {"name": "smith"} -string_smith enum_smith 6.7 {"name": "smith"} {"name": "smith"} +string_smith enum_smith 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D +string_smith enum_smith 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D +string_smith enum_smith 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D +string_smith enum_smith 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D -- !test_75 -- 20230208203804485 20230208203804485_0_0 AA1 07eeba07-b04d-42b5-9e31-35b1a22ea31d-0_0-89-83_20230208203804485.parquet AA1 20 1 1 2021-01-01 00:00:00.000000+08:00 @@ -871,12 +871,12 @@ uuid text Yes false \N NONE -- !test_107 -- \N \N \N -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212 +61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212 +61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212 +61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212 -- !test_107_desc -- decimal_flba decimal(5,2) Yes false \N NONE interval text Yes false \N NONE -uuid varbinary(16) Yes false \N NONE +uuid text Yes false \N NONE diff --git a/regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy b/regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy index e1ad61f64dffee..f983b5cc730060 100644 --- a/regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy +++ b/regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy @@ -57,16 +57,14 @@ suite("test_uuid_orc_varbinary_mapping") { "format"="orc", "backend_id"="${backend.BackendId}", "enable_mapping_varbinary"="${binaryMapping}")""" "order_qt_schema_${scannerV2}_${binaryMapping}" "DESC FUNCTION ${source}" - String uuidValue = binaryMapping ? "CAST(HEX(u) AS UUID)" : "u" - String nestedValue = binaryMapping ? "CAST(HEX(value) AS UUID)" : "value" - String structValue = binaryMapping ? "CAST(HEX(s.k) AS UUID)" : "s.k" - "qt_values_${scannerV2}_${binaryMapping}" """SELECT id, ${uuidValue}, - ARRAY_MAP(value -> ${nestedValue}, a), - ARRAY_MAP(value -> ${nestedValue}, MAP_KEYS(m)), - ARRAY_MAP(value -> ${nestedValue}, MAP_VALUES(m)), - ${structValue}, HEX(b), LENGTH(HEX(b)) + // UUID annotations preserve scalar and nested types independently of the binary option. + "qt_values_${scannerV2}_${binaryMapping}" """SELECT id, u, + ARRAY_MAP(value -> value, a), + ARRAY_MAP(value -> value, MAP_KEYS(m)), + ARRAY_MAP(value -> value, MAP_VALUES(m)), + s.k, HEX(b), LENGTH(HEX(b)) FROM ${source} ORDER BY id""" - "qt_filtered_${scannerV2}_${binaryMapping}" """SELECT id, ${uuidValue}, HEX(b), LENGTH(HEX(b)) + "qt_filtered_${scannerV2}_${binaryMapping}" """SELECT id, u, HEX(b), LENGTH(HEX(b)) FROM ${source} WHERE u IS NOT NULL AND id > 1 ORDER BY id""" } } diff --git a/regression-test/suites/datatype_p0/uuid/test_uuid_parquet_native.groovy b/regression-test/suites/datatype_p0/uuid/test_uuid_parquet_native.groovy index 32567370e980c4..f0675e1b02dba4 100644 --- a/regression-test/suites/datatype_p0/uuid/test_uuid_parquet_native.groovy +++ b/regression-test/suites/datatype_p0/uuid/test_uuid_parquet_native.groovy @@ -50,11 +50,11 @@ suite("test_uuid_parquet_native", "p0,external") { DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')""" sql """INSERT INTO uuid_parquet_native SELECT id,CAST(u AS UUID),CAST(a AS ARRAY), CAST(s AS STRUCT) FROM ${source}""" - // Alternate mappings of the same file to exercise parsed-footer cache isolation. + // A raw-binary option must not override the UUID annotation in the same cached footer. String binarySource = source.replace("'format'='parquet'", "'format'='parquet','enable_mapping_varbinary'='true'") qt_binary_schema "DESC FUNCTION ${binarySource}" - qt_binary_values """SELECT id,HEX(u),CAST(HEX(u) AS UUID) FROM ${binarySource} + qt_binary_values """SELECT id,u,CAST(u AS UUID) FROM ${binarySource} WHERE id < 12 ORDER BY id""" qt_external_schema "DESC FUNCTION ${source}" qt_external_values "SELECT id,u,a,s,LENGTH(u),CAST(u AS UUID) FROM ${source} WHERE id < 12 ORDER BY id" diff --git a/regression-test/suites/external_table_p0/hive/test_hive_uuid_binary.groovy b/regression-test/suites/external_table_p0/hive/test_hive_uuid_binary.groovy new file mode 100644 index 00000000000000..d0fe04a20d159c --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_uuid_binary.groovy @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_hive_uuid_binary", "p0,external,hive,external_docker,external_docker_hive") { + if (!context.config.otherConfigs.get("enableHiveTest")?.equalsIgnoreCase("true")) { + return + } + setHivePrefix("hive2") + String host = context.config.otherConfigs.get("externalEnvIp") + String hmsPort = context.config.otherConfigs.get("hive2HmsPort") + String hdfsPort = context.config.otherConfigs.get("hive2HdfsPort") + String root = "hdfs://${host}:${hdfsPort}/tmp/test_hive_uuid_binary_${UUID.randomUUID()}" + hive_docker "CREATE DATABASE IF NOT EXISTS test_hive_uuid_binary" + sql "DROP CATALOG IF EXISTS hive_uuid_binary" + sql """CREATE CATALOG hive_uuid_binary PROPERTIES ( + "type"="hms", "hive.metastore.uris"="thrift://${host}:${hmsPort}", + "hadoop.username"="doris", "enable.mapping.varbinary"="true")""" + sql "DROP TABLE IF EXISTS hive_uuid_binary_source" + sql """CREATE TABLE hive_uuid_binary_source ( + id INT, u UUID, a ARRAY, m MAP, s STRUCT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO hive_uuid_binary_source VALUES + (1, '00112233445566778899aabbccddeeff', + ['00112233445566778899aabbccddeeff', NULL], + {'00000000000000000000000000000000':'80000000000000000000000000000000'}, + {'80000000000000000000000000000000'}), + (2, '80000000000000000000000000000000', [], {}, {NULL}), + (3, 'ffffffffffffffffffffffffffffffff', NULL, NULL, NULL), + (4, '00000000000000000000000000000000', [NULL], + {'ffffffffffffffffffffffffffffffff':NULL}, {'00000000000000000000000000000000'}), + (5, NULL, NULL, NULL, NULL)""" + for (String format : ["parquet", "orc"]) { + sql """SELECT * FROM hive_uuid_binary_source ORDER BY id + INTO OUTFILE "${root}/${format}/part_" FORMAT AS ${format} + PROPERTIES("fs.defaultFS"="hdfs://${host}:${hdfsPort}", "hadoop.username"="doris")""" + hive_docker "DROP TABLE IF EXISTS test_hive_uuid_binary.uuid_${format}" + // The file carries UUID annotations, but Hive's declared BINARY contract is the raw bytes. + hive_docker """CREATE EXTERNAL TABLE test_hive_uuid_binary.uuid_${format} ( + id INT, u BINARY, a ARRAY, m MAP, s STRUCT) + STORED AS ${format} LOCATION '${root}/${format}'""" + sql "REFRESH CATALOG hive_uuid_binary" + for (boolean scannerV2 : [false, true]) { + sql "SET enable_file_scanner_v2 = ${scannerV2}" + for (boolean strict : [false, true]) { + sql "SET enable_strict_cast = ${strict}" + String table = "hive_uuid_binary.test_hive_uuid_binary.uuid_${format}" + "qt_values_${format}_${scannerV2}_${strict}" """SELECT id, HEX(u), LENGTH(u), + ARRAY_MAP(x -> HEX(x), a), ARRAY_MAP(x -> HEX(x), MAP_KEYS(m)), + ARRAY_MAP(x -> HEX(x), MAP_VALUES(m)), HEX(s.k) + FROM ${table} ORDER BY id""" + "qt_filtered_${format}_${scannerV2}_${strict}" """SELECT id, HEX(u), HEX(a[1]), HEX(s.k) + FROM ${table} WHERE HEX(u) = '00112233445566778899AABBCCDDEEFF' + ORDER BY id""" + } + } + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.groovy new file mode 100644 index 00000000000000..4ec5d37c41cfc5 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.groovy @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_files_timestamp_partition", "p0,external,iceberg") { + if (!context.config.otherConfigs.get("enableIcebergTest")?.toString()?.equalsIgnoreCase("true")) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + def previousZone = sql("SELECT @@time_zone")[0][0] + // Parquet supports both pre-epoch fractions and NTZ. Use identity partitions so the + // metadata rows retain the original microseconds, not a day/hour transform. + spark_iceberg_multi """ + SET spark.sql.session.timeZone=UTC; + CREATE DATABASE IF NOT EXISTS demo.files_timestamp_partition_db; + DROP TABLE IF EXISTS demo.files_timestamp_partition_db.metadata_timestamp_precision; + CREATE TABLE demo.files_timestamp_partition_db.metadata_timestamp_precision ( + id INT, ts TIMESTAMP_NTZ, tz TIMESTAMP + ) USING iceberg PARTITIONED BY (ts, tz) + TBLPROPERTIES ('write.format.default'='parquet'); + INSERT INTO demo.files_timestamp_partition_db.metadata_timestamp_precision + SELECT id, CAST(value AS TIMESTAMP_NTZ), CAST(value AS TIMESTAMP) + FROM VALUES + (0, '1969-12-31 23:59:58.999999'), + (1, '1969-12-31 23:59:59.999999'), + (2, '1970-01-01 00:00:00.000001'), + (3, '1970-01-01 00:00:00.000999'), + (4, '1970-01-01 00:00:00.001000'), + (5, '1970-01-01 00:00:01.234567'), + (6, CAST(NULL AS STRING)) AS input(id, value); + """ + try { + for (boolean mapping : [false, true]) { + sql "DROP CATALOG IF EXISTS test_iceberg_files_timestamp_partition" + sql """CREATE CATALOG test_iceberg_files_timestamp_partition PROPERTIES ( + 'type'='iceberg', 'iceberg.catalog.type'='rest', + 'uri'='http://${host}:${restPort}', + 's3.endpoint'='http://${host}:${minioPort}', + 's3.access_key'='admin', 's3.secret_key'='password', 's3.region'='us-east-1', + 'enable.mapping.timestamp_tz'='${mapping}')""" + sql "SWITCH test_iceberg_files_timestamp_partition" + sql "USE files_timestamp_partition_db" + for (String zone : ["UTC", "Asia/Shanghai"]) { + sql "SET time_zone = '${zone}'" + String tag = "${mapping}_${zone.replace('/', '_')}" + "order_qt_${tag}" """SELECT `partition` + FROM metadata_timestamp_precision\$files ORDER BY file_path""" + // Exercise a pruned struct child as well as the entire partition above. + "order_qt_${tag}_ntz" """SELECT `partition`.ts + FROM metadata_timestamp_precision\$files ORDER BY file_path""" + } + } + } finally { + sql "SET time_zone = '${previousZone}'" + sql "SWITCH internal" + } +} diff --git a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy index 75bd21bc186c50..a807f6d99d62f3 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy @@ -48,6 +48,27 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d "driver_url" = "${driver_url}", "driver_class" = "oracle.jdbc.driver.OracleDriver" );""" + // NUMBER binding cannot represent 1e200, although it is a finite BINARY_DOUBLE. + String binaryTable = "DORIS_BINARY_FLOAT_" + UUID.randomUUID().toString().replace("-", "").substring(0, 8).toUpperCase() + def executeOracle = { statement -> + sql """CALL EXECUTE_STMT("${catalog_name}", "${statement}")""" + } + executeOracle("CREATE TABLE ${binaryTable} (id INT, f BINARY_FLOAT, d BINARY_DOUBLE)") + try { + sql """INSERT INTO ${catalog_name}.${ex_db_name}.${binaryTable} + VALUES (1, CAST(1.25 AS FLOAT), CAST(1e200 AS DOUBLE)), + (2, CAST(-1.25 AS FLOAT), CAST(-1e200 AS DOUBLE)), + (3, CAST(0 AS FLOAT), CAST(1e-200 AS DOUBLE)), + (4, NULL, NULL)""" + def binaryRows = sql """SELECT id, f, d FROM ${catalog_name}.${ex_db_name}.${binaryTable} ORDER BY id""" + assertEquals(4, binaryRows.size()) + assertEquals([1.25d, -1.25d, 0.0d], binaryRows.take(3).collect { it[1].doubleValue() }) + assertEquals([1e200d, -1e200d, 1e-200d], binaryRows.take(3).collect { it[2].doubleValue() }) + assertNull(binaryRows[3][1]) + assertNull(binaryRows[3][2]) + } finally { + executeOracle("DROP TABLE ${binaryTable}") + } order_qt_show_db """ show databases from ${catalog_name}; """ sql """use ${internal_db_name}""" sql """ drop table if exists ${internal_db_name}.${inDorisTable} """ diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog_varbinary.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog_varbinary.groovy index 21f6f2ea7d1e5c..6fa22af15e4da2 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog_varbinary.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog_varbinary.groovy @@ -73,7 +73,7 @@ suite("test_paimon_catalog_varbinary", "p0,external,doris,external_docker,extern qt_varbinary_6 """ select * except(binary_data),length(binary_data) from binary_size_test order by test_id; """ - // Raw TVFs still require explicit binary mapping, independently of catalog mappings. + // Raw TVFs preserve binary bytes even when the obsolete mapping option is absent. qt_varbinary_7 """ select * from hdfs( "uri" = "hdfs://${externalEnvIp}:${hdfs_port}/user/doris/paimon1/db1.db/binary_demo3/bucket-0/data-01367323-fe57-4cf2-8d63-658136eef42a-0.parquet", diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_shredding.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_shredding.groovy index 45cb26295d1d87..c4b80c7a9e0ea1 100644 --- a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_shredding.groovy +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_shredding.groovy @@ -169,8 +169,9 @@ suite("test_paimon_write_variant_shredding", "p0,external,paimon,nonConcurrent") def physicalRows = [] shreddedFiles.each { filePath -> String payloadType = rawPayloadType(filePath) - assertTrue(payloadType.contains("metadata:text")) - assertTrue(payloadType.contains("value:text")) + // Raw TVFs expose the Variant metadata and residual value as opaque binary bytes. + assertTrue(payloadType.contains("metadata:varbinary")) + assertTrue(payloadType.contains("value:varbinary")) assertTrue(payloadType.contains("typed_value:struct")) assertTrue(payloadType.contains("age:struct")) assertTrue(payloadType.contains("profile:struct")) @@ -210,8 +211,8 @@ suite("test_paimon_write_variant_shredding", "p0,external,paimon,nonConcurrent") assertTrue(!unshreddedFiles.isEmpty()) unshreddedFiles.each { filePath -> String payloadType = rawPayloadType(filePath) - assertTrue(payloadType.contains("value:text")) - assertTrue(payloadType.contains("metadata:text")) + assertTrue(payloadType.contains("value:varbinary")) + assertTrue(payloadType.contains("metadata:varbinary")) assertFalse(payloadType.contains("typed_value")) } @@ -257,8 +258,8 @@ suite("test_paimon_write_variant_shredding", "p0,external,paimon,nonConcurrent") assertTrue(!firstInferredFiles.isEmpty()) firstInferredFiles.each { filePath -> String payloadType = rawPayloadType(filePath) - assertTrue(payloadType.contains("metadata:text")) - assertTrue(payloadType.contains("value:text")) + assertTrue(payloadType.contains("metadata:varbinary")) + assertTrue(payloadType.contains("value:varbinary")) assertTrue(payloadType.contains("typed_value:struct")) assertTrue(payloadType.contains("age:struct")) assertTrue(payloadType.contains("profile:struct")) diff --git a/regression-test/suites/external_table_p0/tvf/test_ctas_with_hdfs.groovy b/regression-test/suites/external_table_p0/tvf/test_ctas_with_hdfs.groovy index c02f7258d37b9d..4142259643d711 100644 --- a/regression-test/suites/external_table_p0/tvf/test_ctas_with_hdfs.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_ctas_with_hdfs.groovy @@ -29,6 +29,13 @@ suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { String tableName = "ctas_tvf_1"; def uri = "${defaultFS}" + "/user/doris/preinstalled_data/orc/orc_all_types/p1_col=desktops/p2_col=bigint_col/*" + // Internal tables cannot store VARBINARY. This fixture contains UTF-8 text bytes; + // decode them explicitly so CTAS still tests partitioning and distribution. + String fileColumns = """tinyint_col, smallint_col, int_col, bigint_col, boolean_col, + float_col, double_col, string_col, CAST(binary_col AS STRING) AS binary_col, + timestamp_col, decimal_col, char_col, varchar_col, date_col, + list_double_col, list_string_col""" + // test varchar type as partition column sql """drop table if exists ${tableName}; """ sql """ create table ${tableName} @@ -38,7 +45,7 @@ suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { ) PROPERTIES("replication_num" = "1") as - select * from + select ${fileColumns}, p1_col, p2_col from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", @@ -58,7 +65,7 @@ suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { ) PROPERTIES("replication_num" = "1") as - select * from + select ${fileColumns} from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", @@ -79,7 +86,7 @@ suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { ) PROPERTIES("replication_num" = "1") as - select * from + select ${fileColumns} from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", @@ -100,7 +107,7 @@ suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { ) PROPERTIES("replication_num" = "1") as - select * from + select ${fileColumns} from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", @@ -116,7 +123,7 @@ suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { DISTRIBUTED BY HASH(p1_col) BUCKETS 10 PROPERTIES("replication_num" = "1") as - select * from + select ${fileColumns}, p1_col, p2_col from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy index c91edb5e201ba7..e269dc64823529 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy @@ -198,6 +198,7 @@ suite("test_hdfs_parquet_group2","external,hive,tvf,external_docker") { "format" = "parquet") limit 10; """ + // ENUM is textual despite its BYTE_ARRAY storage; its expected cells must stay strings. uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group2/parquet-thrift-compat.snappy.parquet" order_qt_test_24 """ select * from HDFS( "uri" = "${uri}", diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy index 462d39ad2cbeed..8ab05b46b017e4 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy @@ -574,6 +574,26 @@ suite("test_hdfs_parquet_group6","external,hive,tvf,external_docker") { "format" = "parquet") limit 10; """ + // ENUM is text even though its physical representation is BYTE_ARRAY. + for (boolean mapping : [false, true]) { + String enumSource = """HDFS("uri"="${uri}", "hadoop.username"="${hdfsUserName}", + "format"="parquet", "enable_mapping_varbinary"="${mapping}")""" + def enumSchema = sql "DESC FUNCTION ${enumSource}" + def enumColumn = enumSchema.find { it[0].toString().toLowerCase().contains("enum") } + assertNotNull(enumColumn) + assertEquals("text", enumColumn[1].toString().toLowerCase()) + String enumTable = "parquet_enum_ctas" + sql "DROP TABLE IF EXISTS ${enumTable}" + try { + sql """CREATE TABLE ${enumTable} PROPERTIES("replication_num"="1") AS + SELECT `${enumColumn[0]}` AS enum_value FROM ${enumSource}""" + assertEquals(sql("SELECT `${enumColumn[0]}` FROM ${enumSource} ORDER BY 1"), + sql("SELECT enum_value FROM ${enumTable} ORDER BY 1")) + } finally { + sql "DROP TABLE IF EXISTS ${enumTable}" + } + } + uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group6/07eeba07-b04d-42b5-9e31-35b1a22ea31d-0_0-89-83_20230208203804485.parquet" order_qt_test_75 """ select * from HDFS( "uri" = "${uri}", @@ -736,8 +756,8 @@ suite("test_hdfs_parquet_group6","external,hive,tvf,external_docker") { // The first field is FIXED_LEN_BYTE_ARRAY(16) with a UUID logical annotation. - // V2 decodes it as UUID and casts to the TVF STRING column using canonical UUID text. - // test_107 covers the explicit VARBINARY mapping. + // The UUID annotation preserves canonical text independently of the raw-binary default. + // test_107 verifies that explicitly enabling the option produces the same result. uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group6/fixed_len_byte_array.parquet" order_qt_test_98 """ select * from HDFS( "uri" = "${uri}", diff --git a/regression-test/suites/external_table_p0/tvf/test_tvf_binary_mapping.groovy b/regression-test/suites/external_table_p0/tvf/test_tvf_binary_mapping.groovy new file mode 100644 index 00000000000000..47916fe20af571 --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/test_tvf_binary_mapping.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_tvf_binary_mapping", "external,hive,tvf,external_docker") { + if (!context.config.otherConfigs.get("enableHiveTest")?.toString()?.equalsIgnoreCase("true")) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("hive2HdfsPort") + def originalReader = sql("SELECT @@enable_file_scanner_v2")[0][0] + def fixtures = [ + [format: "orc", path: "test_hdfs_orc/group0/TestOrcFile.test1.orc", column: "bytes1", id: "int1"], + [format: "parquet", path: "test_hdfs_parquet/group0/alltypes_plain.parquet", + column: "string_col", id: "id"] + ] + try { + for (boolean readerV2 : [false, true]) { + sql "SET enable_file_scanner_v2 = ${readerV2}" + for (def fixture : fixtures) { + def baseline = null + for (String legacyValue : ["true", "false", null]) { + String legacyProperty = legacyValue == null ? "" : + ", \"enable_mapping_varbinary\"=\"${legacyValue}\"" + String source = """HDFS( + "uri"="hdfs://${host}:${port}/user/doris/tvf_data/${fixture.path}", + "hadoop.username"="doris", "format"="${fixture.format}"${legacyProperty})""" + def schema = sql "DESC FUNCTION ${source}" + assertTrue(schema.find { it[0] == fixture.column }[1].toString().startsWith("varbinary")) + def rows = sql "SELECT HEX(`${fixture.column}`) FROM ${source} ORDER BY 1" + if (baseline == null) { + baseline = rows + } + // Legacy flags must not change the inferred schema or the stored bytes. + assertEquals(baseline, rows) + if (fixture.format == "orc") { + assertEquals([[""], ["0001020304"]], rows) + } + sql "DROP TABLE IF EXISTS tvf_binary_mapping_ctas" + // Internal tables do not support VARBINARY; CTAS must not silently decode it as text. + test { + sql """CREATE TABLE tvf_binary_mapping_ctas PROPERTIES ("replication_num"="1") + AS SELECT `${fixture.id}` AS id, `${fixture.column}` AS payload FROM ${source}""" + exception "varbinary" + } + sql """CREATE TABLE tvf_binary_mapping_ctas PROPERTIES ("replication_num"="1") + AS SELECT `${fixture.id}` AS id, HEX(`${fixture.column}`) AS payload_hex FROM ${source}""" + assertEquals(rows, sql("SELECT payload_hex FROM tvf_binary_mapping_ctas ORDER BY 1")) + } + } + } + } finally { + sql "DROP TABLE IF EXISTS tvf_binary_mapping_ctas" + sql "SET enable_file_scanner_v2 = ${originalReader}" + } +}