Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion be/src/exprs/function/cast/function_cast.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

#include <utility>

#include "core/column/column_varbinary.h"
#include "core/data_type/data_type_agg_state.h"
#include "core/data_type/data_type_decimal.h"
#include "core/data_type/data_type_number.h" // IWYU pragma: keep
Expand All @@ -26,6 +27,7 @@
#include "core/data_type/data_type_variant.h"
#include "core/data_type/data_type_variant_v2.h"
#include "core/data_type/primitive_type.h"
#include "core/value/uuid_value.h"
#include "exprs/function/cast/cast_to_array.h"
#include "exprs/function/cast/cast_to_jsonb.h"
#include "exprs/function/cast/cast_to_map.h"
Expand Down Expand Up @@ -65,7 +67,26 @@ WrapperType create_varbinary_wrapper(const DataTypePtr& from_type_untyped) {
return cast_from_string_to_generic;
}

return CastWrapper::create_unsupport_wrapper("Cast to Varbinary only support from String type");
if (from_type_untyped->get_primitive_type() == TYPE_UUID) {
return [](FunctionContext*, Block& block, const ColumnNumbers& arguments, uint32_t result,
size_t rows, const NullMap::value_type*) {
const auto& source =
assert_cast<const ColumnUUID&>(*block.get_by_position(arguments[0]).column);
auto binary = ColumnVarbinary::create();
binary->reserve(rows);
for (size_t i = 0; i < rows; ++i) {
// File Scanner V2 decodes UUIDs before mapping Hive BINARY columns. Recreate the
// file's big-endian bytes, not the host integer layout or canonical UUID text.
const auto bytes = UUIDValue::to_big_endian(source.get_data()[i]);
binary->insert_data(reinterpret_cast<const char*>(bytes.data()), bytes.size());
}
block.get_by_position(result).column = std::move(binary);
return Status::OK();
};
}

return CastWrapper::create_unsupport_wrapper(
"Cast to Varbinary only support from String or UUID type");
}

WrapperType prepare_unpack_dictionaries(FunctionContext* context, const DataTypePtr& from_type,
Expand Down
17 changes: 12 additions & 5 deletions be/src/format/orc/vorc_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1736,13 +1736,14 @@ DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) {
case orc::TypeKind::STRING:
return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true);
case orc::TypeKind::BINARY:
if (_scan_params.__isset.enable_mapping_varbinary &&
_scan_params.enable_mapping_varbinary) {
// Logical UUID annotations take precedence over the default for raw binary fields.
if (orc_type->hasAttributeKey("doris.logical_type") &&
orc_type->getAttributeValue("doris.logical_type") == "uuid") {
return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_UUID, true);
} else if (_scan_params.__isset.enable_mapping_varbinary &&
_scan_params.enable_mapping_varbinary) {
return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_VARBINARY,
true);
} else if (orc_type->hasAttributeKey("doris.logical_type") &&
orc_type->getAttributeValue("doris.logical_type") == "uuid") {
return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_UUID, true);
} else {
return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true);
}
Expand Down Expand Up @@ -2414,6 +2415,12 @@ Status OrcReader::_orc_column_to_doris_column(
MutableColumnPtr data_column;
if (orc_column_type != nullptr) {
auto src_type = convert_to_doris_type(orc_column_type);
if (src_type->get_primitive_type() == TYPE_UUID &&
data_type->get_primitive_type() == TYPE_VARBINARY) {
// Hive BINARY keeps the file bytes even when the ORC carrier has a UUID annotation.
// Decode that carrier directly, including recursively visited complex-type children.
src_type = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, true);
}
bool is_dict_filter_col = false;
for (const std::pair<std::string, int>& dict_col : _dict_filter_cols) {
if (col_name == dict_col.first) {
Expand Down
7 changes: 4 additions & 3 deletions be/src/format/parquet/parquet_column_convert.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -261,9 +261,10 @@ std::unique_ptr<PhysicalToLogicalConverter> PhysicalToLogicalConverter::get_conv
} else if (is_parquet_native_type(src_logical_primitive)) {
bool is_string_logical_type = is_string_type(src_logical_primitive);
if (is_string_logical_type && src_physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY) {
if (parquet_schema.logicalType.__isset.UUID && !preserve_binary_uuid) {
// Ordinary Parquet scans expose canonical text, as in File Scanner V2.
// Iceberg retains its raw 16-byte STRING carrier for equality deletes/defaults.
if (parquet_schema.logicalType.__isset.UUID && !preserve_binary_uuid &&
dst_logical_type->get_primitive_type() != TYPE_VARBINARY) {
// Inferred STRING fields expose canonical text. Explicit BINARY targets and
// Iceberg's equality-delete carrier must retain the original 16 bytes.
if (parquet_schema.type_length != UUIDValue::BINARY_LENGTH) {
physical_converter = std::make_unique<UnsupportedConverter>(src_physical_type,
src_logical_type);
Expand Down
12 changes: 5 additions & 7 deletions be/src/format/parquet/schema_desc.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,8 @@ std::pair<DataTypePtr, bool> FieldDescriptor::convert_to_doris_type(
tparquet::LogicalType logicalType, bool nullable) {
std::pair<DataTypePtr, bool> ans = {std::make_shared<DataTypeNothing>(), false};
bool& is_type_compatibility = ans.second;
if (logicalType.__isset.STRING) {
// ENUM is UTF-8 text, not an unannotated binary payload.
if (logicalType.__isset.STRING || logicalType.__isset.ENUM) {
ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable);
} else if (logicalType.__isset.DECIMAL) {
ans.first = DataTypeFactory::instance().create_data_type(TYPE_DECIMAL128I, nullable,
Expand Down Expand Up @@ -316,12 +317,8 @@ std::pair<DataTypePtr, bool> FieldDescriptor::convert_to_doris_type(
} else if (logicalType.__isset.JSON) {
ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable);
} else if (logicalType.__isset.UUID) {
if (_enable_mapping_varbinary) {
ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable, -1,
-1, 16);
} else {
ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable);
}
// The raw-binary default must not override UUID's existing canonical-text contract.
ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable);
Comment thread
Gabriel39 marked this conversation as resolved.
Resolved
} else if (logicalType.__isset.FLOAT16) {
ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable);
} else {
Expand All @@ -336,6 +333,7 @@ std::pair<DataTypePtr, bool> FieldDescriptor::convert_to_doris_type(
bool& is_type_compatibility = ans.second;
switch (physical_schema.converted_type) {
case tparquet::ConvertedType::type::UTF8:
case tparquet::ConvertedType::type::ENUM:
ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable);
break;
case tparquet::ConvertedType::type::DECIMAL:
Expand Down
9 changes: 5 additions & 4 deletions be/src/format_v2/orc/orc_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1131,11 +1131,12 @@ DataTypePtr OrcReader::_convert_to_doris_type(const ::orc::Type& type) const {
data_type = std::make_shared<DataTypeString>();
break;
case ::orc::TypeKind::BINARY:
if (_enable_mapping_varbinary) {
data_type = std::make_shared<DataTypeVarbinary>();
} else if (type.hasAttributeKey("doris.logical_type") &&
type.getAttributeValue("doris.logical_type") == "uuid") {
// A logical UUID remains a UUID when the raw-binary default is enabled.
if (type.hasAttributeKey("doris.logical_type") &&
type.getAttributeValue("doris.logical_type") == "uuid") {
data_type = std::make_shared<DataTypeUUID>();
} else if (_enable_mapping_varbinary) {
data_type = std::make_shared<DataTypeVarbinary>();
} else {
data_type = std::make_shared<DataTypeString>();
}
Expand Down
11 changes: 3 additions & 8 deletions be/src/format_v2/parquet/native_schema_desc.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -891,14 +891,9 @@ std::pair<DataTypePtr, bool> NativeFieldDescriptor::convert_to_doris_type(
ans.first = DataTypeFactory::instance().create_data_type(
TYPE_DATETIMEV2, nullable, 0, logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6);
} else if (logicalType.__isset.UUID) {
// Keep binary UUID carriers consistent with table defaults and other physical formats.
// Ordinary Parquet scans still expose the native UUID unless VARBINARY was requested.
auto uuid_type = TYPE_UUID;
if (_enable_mapping_varbinary) {
uuid_type = TYPE_VARBINARY;
} else if (_preserve_binary_uuid) {
uuid_type = TYPE_STRING;
}
// UUID is a logical type, so the raw-binary default must not override it.
// Preserve table-requested raw carriers for Iceberg equality deletes and defaults.
auto uuid_type = _preserve_binary_uuid ? TYPE_STRING : TYPE_UUID;
ans.first = DataTypeFactory::instance().create_data_type(uuid_type, nullable, -1, -1, 16);
} else if (logicalType.__isset.FLOAT16) {
ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable);
Expand Down
92 changes: 92 additions & 0 deletions be/test/exprs/function/cast/cast_uuid_binary_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "core/block/block.h"
#include "core/column/column_array.h"
#include "core/column/column_nullable.h"
#include "core/column/column_varbinary.h"
#include "core/column/column_vector.h"
#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_uuid.h"
#include "core/data_type/data_type_varbinary.h"
#include "core/value/uuid_value.h"
#include "exprs/function/simple_function_factory.h"
#include "exprs/function_context.h"
#include "gtest/gtest.h"
#include "runtime/runtime_state.h"

namespace doris {

TEST(UuidBinaryCastTest, PreservesFileBytesAndNestedNulls) {
for (bool strict : {false, true}) {
RuntimeState state;
for (bool nested : {false, true}) {
const std::vector<std::string> bytes = {std::string(16, '\0'), std::string(16, '\xff'),
std::string("\x80", 1) + std::string(15, '\0'),
"abcDeFGhijkLmnOp"};
auto values = ColumnUUID::create();
auto nulls = ColumnUInt8::create();
for (const auto& value : bytes) {
values->insert_value(
UUIDValue::from_big_endian(reinterpret_cast<const uint8_t*>(value.data())));
nulls->insert_value(0);
}
values->insert_default();
nulls->insert_value(1);
ColumnPtr source = ColumnNullable::create(std::move(values), std::move(nulls));
DataTypePtr source_type = make_nullable(std::make_shared<DataTypeUUID>());
DataTypePtr target_type = make_nullable(std::make_shared<DataTypeVarbinary>());
if (nested) {
auto offsets = ColumnArray::ColumnOffsets::create();
offsets->insert_value(3);
offsets->insert_value(3);
offsets->insert_value(5);
source = ColumnArray::create(source, std::move(offsets));
source_type = std::make_shared<DataTypeArray>(source_type);
target_type = std::make_shared<DataTypeArray>(target_type);
}
auto context = FunctionContext::create_context(&state, {}, {});
context->set_enable_strict_mode(strict);
Block block;
block.insert({source, source_type, "uuid"});
auto function = SimpleFunctionFactory::instance().get_function(
"CAST", {{source, source_type, "uuid"}}, target_type);
ASSERT_NE(function, nullptr);
const int result_id = block.columns();
block.insert({nullptr, target_type, "binary"});
auto status = function->execute(context.get(), block, {0}, result_id, source->size());
ASSERT_TRUE(status.ok()) << status;
ColumnPtr result = block.get_by_position(result_id).column;
if (nested) {
const auto& array = assert_cast<const ColumnArray&>(*result);
EXPECT_EQ(array.get_offsets(),
assert_cast<const ColumnArray&>(*source).get_offsets());
result = array.get_data_ptr();
}
const auto& nullable = assert_cast<const ColumnNullable&>(*result);
EXPECT_EQ(nullable.size(), 5);
for (size_t i = 0; i < bytes.size(); ++i) {
EXPECT_FALSE(nullable.is_null_at(i));
EXPECT_EQ(nullable.get_nested_column().get_data_at(i).to_string(), bytes[i]);
}
EXPECT_TRUE(nullable.is_null_at(4));
}
}
}

} // namespace doris
13 changes: 8 additions & 5 deletions be/test/format/parquet/parquet_column_convert_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,14 @@ TEST(ParquetColumnConvertTest, UuidTextAndBinaryCarriersPreserveNullsAcrossBatch
"00000000-0000-0000-0000-000000000000", "7f7f7f7f-7f7f-7f7f-7f7f-7f7f7f7f7f7f",
"80000000-0000-0000-0000-000000000000", "ffffffff-ffff-ffff-ffff-ffffffffffff",
"61626344-6546-4768-696a-6b4c6d6e4f70", "00000000-0000-0000-0000-000000000000"};
for (const bool preserve_binary : {false, true}) {
for (int carrier = 0; carrier < 3; ++carrier) {
const bool preserve_binary = carrier == 1;
const auto dst_type = DataTypeFactory::instance().create_data_type(
carrier == 2 ? TYPE_VARBINARY : TYPE_STRING, true);
auto converter = PhysicalToLogicalConverter::get_converter(
&field, field.data_type, field.data_type, nullptr, false, preserve_binary);
&field, field.data_type, dst_type, nullptr, false, preserve_binary);
ASSERT_TRUE(converter->support());
ColumnPtr dst = field.data_type->create_column();
ColumnPtr dst = dst_type->create_column();
for (size_t batch = 0; batch < 2; ++batch) {
auto values = ColumnFixedLengthObject::create(16);
auto nulls = ColumnUInt8::create();
Expand All @@ -76,14 +79,14 @@ TEST(ParquetColumnConvertTest, UuidTextAndBinaryCarriersPreserveNullsAcrossBatch
nulls->insert_value(i == 5);
}
ColumnPtr src = ColumnNullable::create(std::move(values), std::move(nulls));
ASSERT_TRUE(converter->convert(src, field.data_type, field.data_type, dst, false).ok());
ASSERT_TRUE(converter->convert(src, field.data_type, dst_type, dst, false).ok());
EXPECT_EQ(dst->size(), (batch + 1) * 3);
}
const auto& nullable = assert_cast<const ColumnNullable&>(*dst);
for (size_t i = 0; i < raw_values.size(); ++i) {
EXPECT_EQ(nullable.get_null_map_data()[i], i == 5);
EXPECT_EQ(nullable.get_nested_column().get_data_at(i).to_string(),
preserve_binary ? raw_values[i] : text_values[i]);
carrier != 0 ? raw_values[i] : text_values[i]);
}
}
}
Expand Down
27 changes: 9 additions & 18 deletions be/test/format/parquet/parquet_reader_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -369,14 +369,14 @@ TEST_F(ParquetReaderTest, normal) {
delete p_reader;
}

TEST_F(ParquetReaderTest, uuid_text_and_varbinary_cached_mappings) {
// Reuse the same footer cache with both mappings, then repeat each cache lookup.
TEST_F(ParquetReaderTest, uuid_annotation_preserves_text_across_mapping_flags) {
// Changing the raw-binary option must not change a cached UUID field into VARBINARY.
for (const bool mapping : {false, true, false, true}) {
TDescriptorTable t_desc_table;
TTableDescriptor t_table_desc;
std::vector<std::string> table_column_names = {"id", "col1"};
std::vector<TPrimitiveType::type> table_column_types = {
TPrimitiveType::INT, mapping ? TPrimitiveType::VARBINARY : TPrimitiveType::STRING};
std::vector<TPrimitiveType::type> table_column_types = {TPrimitiveType::INT,
TPrimitiveType::STRING};
create_table_desc(t_desc_table, t_table_desc, table_column_names, table_column_types);
DescriptorTbl* desc_tbl;
ObjectPool obj_pool;
Expand Down Expand Up @@ -438,20 +438,11 @@ TEST_F(ParquetReaderTest, uuid_text_and_varbinary_cached_mappings) {
}
auto col = block->safe_get_by_position(1).column;
auto nullable_column = assert_cast<const ColumnNullable*>(col.get());
if (mapping) {
auto varbinary_column = assert_cast<const ColumnVarbinary*>(
nullable_column->get_nested_column_ptr().get());
auto& data = varbinary_column->get_data();
EXPECT_EQ(data[0].dump_hex(), "0x550E8400E29B41D4A716446655440000");
EXPECT_EQ(data[1].dump_hex(), "0x123E4567E89B12D3A456426614174000");
EXPECT_EQ(data[2].dump_hex(), "0x00000000000000000000000000000000");
} else {
const auto& strings =
assert_cast<const ColumnString&>(nullable_column->get_nested_column());
EXPECT_EQ(strings.get_data_at(0).to_string(), "550e8400-e29b-41d4-a716-446655440000");
EXPECT_EQ(strings.get_data_at(1).to_string(), "123e4567-e89b-12d3-a456-426614174000");
EXPECT_EQ(strings.get_data_at(2).to_string(), "00000000-0000-0000-0000-000000000000");
}
const auto& strings =
assert_cast<const ColumnString&>(nullable_column->get_nested_column());
EXPECT_EQ(strings.get_data_at(0).to_string(), "550e8400-e29b-41d4-a716-446655440000");
EXPECT_EQ(strings.get_data_at(1).to_string(), "123e4567-e89b-12d3-a456-426614174000");
EXPECT_EQ(strings.get_data_at(2).to_string(), "00000000-0000-0000-0000-000000000000");
}
}

Expand Down
Loading
Loading