From 3feb3805c3a72491c797b33f4ceb66e8e05694af Mon Sep 17 00:00:00 2001 From: kaka11chen Date: Thu, 27 Jun 2024 09:38:45 +0800 Subject: [PATCH] [Test](iceberg-writer) Add iceberg partition transform unit tests and fix some issues. --- .../writer/iceberg/partition_transformers.cpp | 8 +- .../writer/iceberg/partition_transformers.h | 105 ++-- .../iceberg/partition_transformers_test.cpp | 489 ++++++++++++++++++ 3 files changed, 545 insertions(+), 57 deletions(-) create mode 100644 be/test/vec/sink/writer/iceberg/partition_transformers_test.cpp diff --git a/be/src/vec/sink/writer/iceberg/partition_transformers.cpp b/be/src/vec/sink/writer/iceberg/partition_transformers.cpp index 6b2a3305b9e263..ee8268d30f73b5 100644 --- a/be/src/vec/sink/writer/iceberg/partition_transformers.cpp +++ b/be/src/vec/sink/writer/iceberg/partition_transformers.cpp @@ -25,16 +25,16 @@ namespace doris { namespace vectorized { -const std::chrono::time_point PartitionColumnTransformUtils::EPOCH = - std::chrono::system_clock::from_time_t(0); +const std::chrono::sys_days PartitionColumnTransformUtils::EPOCH = std::chrono::sys_days( + std::chrono::year {1970} / std::chrono::January / std::chrono::day {1}); std::unique_ptr PartitionColumnTransforms::create( const doris::iceberg::PartitionField& field, const TypeDescriptor& source_type) { auto& transform = field.transform(); - static const std::regex hasWidth(R"((\w+)\[(\d+)\])"); + static const std::regex has_width(R"((\w+)\[(\d+)\])"); std::smatch width_match; - if (std::regex_match(transform, width_match, hasWidth)) { + if (std::regex_match(transform, width_match, has_width)) { std::string name = width_match[1]; int parsed_width = std::stoi(width_match[2]); diff --git a/be/src/vec/sink/writer/iceberg/partition_transformers.h b/be/src/vec/sink/writer/iceberg/partition_transformers.h index 7db8bfb1886709..84ee3029cdd18f 100644 --- a/be/src/vec/sink/writer/iceberg/partition_transformers.h +++ b/be/src/vec/sink/writer/iceberg/partition_transformers.h @@ -66,16 +66,12 @@ class PartitionColumnTransformUtils { } static std::string human_year(int year_ordinal) { - auto year = std::chrono::year_month_day( - std::chrono::sys_days(std::chrono::floor( - EPOCH + std::chrono::years(year_ordinal)))) - .year(); - return std::to_string(static_cast(year)); + auto ymd = std::chrono::year_month_day {EPOCH} + std::chrono::years(year_ordinal); + return std::to_string(static_cast(ymd.year())); } static std::string human_month(int month_ordinal) { - auto ymd = std::chrono::year_month_day(std::chrono::sys_days( - std::chrono::floor(EPOCH + std::chrono::months(month_ordinal)))); + auto ymd = std::chrono::year_month_day {EPOCH} + std::chrono::months(month_ordinal); return fmt::format("{:04d}-{:02d}", static_cast(ymd.year()), static_cast(ymd.month())); } @@ -98,7 +94,7 @@ class PartitionColumnTransformUtils { } private: - static const std::chrono::time_point EPOCH; + static const std::chrono::sys_days EPOCH; PartitionColumnTransformUtils() = default; }; @@ -112,7 +108,7 @@ class PartitionColumnTransform { virtual const TypeDescriptor& get_result_type() const = 0; - virtual ColumnWithTypeAndName apply(Block& block, int column_pos) = 0; + virtual ColumnWithTypeAndName apply(const Block& block, int column_pos) = 0; virtual std::string to_human_string(const TypeDescriptor& type, const std::any& value) const; @@ -129,7 +125,7 @@ class IdentityPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _source_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); return {column_with_type_and_name.column, column_with_type_and_name.type, column_with_type_and_name.name}; @@ -148,9 +144,9 @@ class StringTruncatePartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _source_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { + static_cast(_width); auto int_type = std::make_shared(); - size_t num_columns_without_result = block.columns(); const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr string_column_ptr; @@ -165,29 +161,31 @@ class StringTruncatePartitionColumnTransform : public PartitionColumnTransform { string_column_ptr = column_with_type_and_name.column; is_nullable = false; } - block.replace_by_position(column_pos, std::move(string_column_ptr)); - block.insert( - {int_type->create_column_const(block.rows(), to_field(1)), int_type, "const 1"}); - block.insert({int_type->create_column_const(block.rows(), to_field(_width)), int_type, - fmt::format("const {}", _width)}); - block.insert({nullptr, std::make_shared(), "result"}); + + // Create a temp_block to execute substring function. + Block temp_block; + temp_block.insert(column_with_type_and_name); + temp_block.insert({int_type->create_column_const(temp_block.rows(), to_field(1)), int_type, + "const 1"}); + temp_block.insert({int_type->create_column_const(temp_block.rows(), to_field(_width)), + int_type, fmt::format("const {}", _width)}); + temp_block.insert({nullptr, std::make_shared(), "result"}); ColumnNumbers temp_arguments(3); - temp_arguments[0] = column_pos; // str column - temp_arguments[1] = num_columns_without_result; // pos - temp_arguments[2] = num_columns_without_result + 1; // width - size_t result_column_id = num_columns_without_result + 2; + temp_arguments[0] = 0; // str column + temp_arguments[1] = 1; // pos + temp_arguments[2] = 2; // width + size_t result_column_id = 3; - SubstringUtil::substring_execute(block, temp_arguments, result_column_id, block.rows()); + SubstringUtil::substring_execute(temp_block, temp_arguments, result_column_id, + temp_block.rows()); if (is_nullable) { - auto res_column = ColumnNullable::create(block.get_by_position(result_column_id).column, - null_map_column_ptr); - Block::erase_useless_column(&block, num_columns_without_result); + auto res_column = ColumnNullable::create( + temp_block.get_by_position(result_column_id).column, null_map_column_ptr); return {std::move(res_column), DataTypeFactory::instance().create_data_type(get_result_type(), true), column_with_type_and_name.name}; } else { - auto res_column = block.get_by_position(result_column_id).column; - Block::erase_useless_column(&block, num_columns_without_result); + auto res_column = temp_block.get_by_position(result_column_id).column; return {std::move(res_column), DataTypeFactory::instance().create_data_type(get_result_type(), false), column_with_type_and_name.name}; @@ -208,7 +206,7 @@ class IntegerTruncatePartitionColumnTransform : public PartitionColumnTransform const TypeDescriptor& get_result_type() const override { return _source_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -267,7 +265,7 @@ class BigintTruncatePartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _source_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -327,7 +325,7 @@ class DecimalTruncatePartitionColumnTransform : public PartitionColumnTransform const TypeDescriptor& get_result_type() const override { return _source_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr; @@ -388,7 +386,7 @@ class IntBucketPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -417,7 +415,8 @@ class IntBucketPartitionColumnTransform : public PartitionColumnTransform { while (p_in < end_in) { Int64 long_value = static_cast(*p_in); uint32_t hash_value = HashUtil::murmur_hash3_32(&long_value, sizeof(long_value), 0); - *p_out = ((hash_value >> 1) & INT32_MAX) % _bucket_num; + // *p_out = ((hash_value >> 1) & INT32_MAX) % _bucket_num; + *p_out = (hash_value & INT32_MAX) % _bucket_num; ++p_in; ++p_out; } @@ -450,7 +449,7 @@ class BigintBucketPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -469,16 +468,17 @@ class BigintBucketPartitionColumnTransform : public PartitionColumnTransform { const auto& in_data = assert_cast(column_ptr.get())->get_data(); //3) do partition routing - auto col_res = ColumnInt64::create(); - ColumnInt64::Container& out_data = col_res->get_data(); + auto col_res = ColumnInt32::create(); + ColumnInt32::Container& out_data = col_res->get_data(); out_data.resize(in_data.size()); const Int64* end_in = in_data.data() + in_data.size(); const Int64* __restrict p_in = in_data.data(); - Int64* __restrict p_out = out_data.data(); + Int32* __restrict p_out = out_data.data(); while (p_in < end_in) { Int64 long_value = static_cast(*p_in); uint32_t hash_value = HashUtil::murmur_hash3_32(&long_value, sizeof(long_value), 0); - int value = ((hash_value >> 1) & INT32_MAX) % _bucket_num; + // int value = ((hash_value >> 1) & INT32_MAX) % _bucket_num; + int value = (hash_value & INT32_MAX) % _bucket_num; *p_out = value; ++p_in; ++p_out; @@ -513,7 +513,7 @@ class DecimalBucketPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -546,7 +546,7 @@ class DecimalBucketPartitionColumnTransform : public PartitionColumnTransform { while (p_in < end_in) { std::string buffer = BitUtil::IntToByteBuffer(*p_in); uint32_t hash_value = HashUtil::murmur_hash3_32(buffer.data(), buffer.size(), 0); - *p_out = ((hash_value >> 1) & INT32_MAX) % _bucket_num; + *p_out = (hash_value & INT32_MAX) % _bucket_num; ++p_in; ++p_out; } @@ -592,13 +592,12 @@ class DateBucketPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); CHECK(column_ptr != nullptr); - //2) get the input data from block //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; @@ -628,7 +627,7 @@ class DateBucketPartitionColumnTransform : public PartitionColumnTransform { Int64 long_value = static_cast(days_from_unix_epoch); uint32_t hash_value = HashUtil::murmur_hash3_32(&long_value, sizeof(long_value), 0); - *p_out = ((hash_value >> 1) & INT32_MAX) % _bucket_num; + *p_out = (hash_value & INT32_MAX) % _bucket_num; ++p_in; ++p_out; } @@ -661,7 +660,7 @@ class TimestampBucketPartitionColumnTransform : public PartitionColumnTransform const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -742,7 +741,7 @@ class StringBucketPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -807,7 +806,7 @@ class DateYearPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -877,7 +876,7 @@ class TimestampYearPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -947,7 +946,7 @@ class DateMonthPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -1017,7 +1016,7 @@ class TimestampMonthPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -1087,7 +1086,7 @@ class DateDayPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -1163,7 +1162,7 @@ class TimestampDayPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -1238,7 +1237,7 @@ class TimestampHourPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { //1) get the target column ptr const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr = column_with_type_and_name.column->convert_to_full_column_if_const(); @@ -1308,7 +1307,7 @@ class VoidPartitionColumnTransform : public PartitionColumnTransform { const TypeDescriptor& get_result_type() const override { return _target_type; } - ColumnWithTypeAndName apply(Block& block, int column_pos) override { + ColumnWithTypeAndName apply(const Block& block, int column_pos) override { const ColumnWithTypeAndName& column_with_type_and_name = block.get_by_position(column_pos); ColumnPtr column_ptr; diff --git a/be/test/vec/sink/writer/iceberg/partition_transformers_test.cpp b/be/test/vec/sink/writer/iceberg/partition_transformers_test.cpp new file mode 100644 index 00000000000000..205b1526839ffe --- /dev/null +++ b/be/test/vec/sink/writer/iceberg/partition_transformers_test.cpp @@ -0,0 +1,489 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/sink/writer/iceberg/partition_transformers.h" + +#include + +#include "vec/data_types/data_type_time_v2.h" + +namespace doris::vectorized { + +class PartitionTransformersTest : public testing::Test { +public: + PartitionTransformersTest() = default; + virtual ~PartitionTransformersTest() = default; +}; + +TEST_F(PartitionTransformersTest, test_integer_truncate_transform) { + const std::vector values({1, -1}); + auto column = ColumnInt32::create(); + column->insert_many_fix_len_data(reinterpret_cast(values.data()), values.size()); + ColumnWithTypeAndName test_int(column->get_ptr(), std::make_shared(), + "test_int"); + + Block block({test_int}); + TypeDescriptor source_type(PrimitiveType::TYPE_INT); + IntegerTruncatePartitionColumnTransform transform(source_type, 10); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {0, -10}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + } +} + +TEST_F(PartitionTransformersTest, test_bigint_truncate_transform) { + const std::vector values({1, -1}); + auto column = ColumnInt64::create(); + column->insert_many_fix_len_data(reinterpret_cast(values.data()), values.size()); + ColumnWithTypeAndName test_bigint(column->get_ptr(), std::make_shared(), + "test_bigint"); + + Block block({test_bigint}); + TypeDescriptor source_type(PrimitiveType::TYPE_BIGINT); + BigintTruncatePartitionColumnTransform transform(source_type, 10); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {0, -10}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + } +} + +TEST_F(PartitionTransformersTest, test_decimal32_truncate_transform) { + const std::vector values({1065}); + auto column = ColumnDecimal32::create(0, 2); + column->insert_many_fix_len_data(reinterpret_cast(values.data()), values.size()); + ColumnWithTypeAndName test_decimal32(column->get_ptr(), + std::make_shared>(4, 2), + "test_decimal32"); + + Block block({test_decimal32}); + TypeDescriptor source_type = TypeDescriptor::create_decimalv3_type(4, 2); + DecimalTruncatePartitionColumnTransform transform(source_type, 50); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {1050}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i].value); + } +} + +TEST_F(PartitionTransformersTest, test_string_truncate_transform) { + const std::vector values({{"iceberg", sizeof("iceberg") - 1}}); + auto column = ColumnString::create(); + column->insert_many_strings(&values[0], values.size()); + ColumnWithTypeAndName test_string(column->get_ptr(), std::make_shared(), + "test_string"); + + Block block({test_string}); + TypeDescriptor source_type = TypeDescriptor::create_string_type(); + StringTruncatePartitionColumnTransform transform(source_type, 3); + + auto result = transform.apply(block, 0); + const auto result_column = assert_cast(result.column.get()); + const char result_data[] = {'i', 'c', 'e'}; + std::vector expected_data = { + {result_data, sizeof(result_data) / sizeof(result_data[0])}}; + EXPECT_EQ(expected_data.size(), result_column->size()); + for (size_t i = 0; i < result_column->size(); ++i) { + EXPECT_EQ(expected_data[i], result_column->get_data_at(i)); + } +} + +TEST_F(PartitionTransformersTest, test_integer_bucket_transform) { + const std::vector values({34, -123}); // 2017239379, -471378254 + auto column = ColumnInt32::create(); + column->insert_many_fix_len_data(reinterpret_cast(values.data()), values.size()); + ColumnWithTypeAndName test_int(column->get_ptr(), std::make_shared(), + "test_int"); + + Block block({test_int}); + TypeDescriptor source_type(PrimitiveType::TYPE_INT); + IntBucketPartitionColumnTransform transform(source_type, 16); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {3, 2}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + } +} + +TEST_F(PartitionTransformersTest, test_bigint_bucket_transform) { + const std::vector values({34, -123}); // 2017239379, -471378254 + auto column = ColumnInt64::create(); + column->insert_many_fix_len_data(reinterpret_cast(values.data()), values.size()); + ColumnWithTypeAndName test_bigint(column->get_ptr(), std::make_shared(), + "test_bigint"); + + Block block({test_bigint}); + TypeDescriptor source_type(PrimitiveType::TYPE_BIGINT); + BigintBucketPartitionColumnTransform transform(source_type, 16); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {3, 2}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + } +} + +TEST_F(PartitionTransformersTest, test_decimal32_bucket_transform) { + const std::vector values({1420}); // -500754589 + auto column = ColumnDecimal32::create(0, 2); + column->insert_many_fix_len_data(reinterpret_cast(values.data()), values.size()); + ColumnWithTypeAndName test_decimal32(column->get_ptr(), + std::make_shared>(4, 2), + "test_decimal32"); + + Block block({test_decimal32}); + TypeDescriptor source_type = TypeDescriptor::create_decimalv3_type(4, 2); + DecimalBucketPartitionColumnTransform transform(source_type, 16); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {3}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + } +} + +TEST_F(PartitionTransformersTest, test_date_bucket_transform) { + auto column = ColumnDateV2::create(); + auto& date_v2_data = column->get_data(); + DateV2Value value; + value.unchecked_set_time(2017, 11, 16, 0, 0, 0, 0); // -653330422 + date_v2_data.push_back(*reinterpret_cast(&value)); + ColumnWithTypeAndName test_date(column->get_ptr(), std::make_shared(), + "test_date"); + + Block block({test_date}); + TypeDescriptor source_type(PrimitiveType::TYPE_DATEV2); + DateBucketPartitionColumnTransform transform(source_type, 16); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {10}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + } +} + +TEST_F(PartitionTransformersTest, test_timestamp_bucket_transform) { + auto column = ColumnDateTimeV2::create(); + auto& datetime_v2_data = column->get_data(); + DateV2Value value; + value.unchecked_set_time(2017, 11, 16, 22, 31, 8, 0); // -2047944441 + datetime_v2_data.push_back(*reinterpret_cast(&value)); + ColumnWithTypeAndName test_timestamp(column->get_ptr(), std::make_shared(), + "test_timestamp"); + + Block block({test_timestamp}); + TypeDescriptor source_type(PrimitiveType::TYPE_DATETIMEV2); + TimestampBucketPartitionColumnTransform transform(source_type, 16); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {7}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + } +} + +TEST_F(PartitionTransformersTest, test_string_bucket_transform) { + const std::vector values({{"iceberg", sizeof("iceberg") - 1}}); // 1210000089 + auto column = ColumnString::create(); + column->insert_many_strings(&values[0], values.size()); + ColumnWithTypeAndName test_string(column->get_ptr(), std::make_shared(), + "test_string"); + + Block block({test_string}); + TypeDescriptor source_type(PrimitiveType::TYPE_STRING); + StringBucketPartitionColumnTransform transform(source_type, 16); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {9}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + } +} + +TEST_F(PartitionTransformersTest, test_date_year_transform) { + auto column = ColumnDateV2::create(); + auto& date_v2_data = column->get_data(); + DateV2Value value; + value.unchecked_set_time(2017, 11, 16, 0, 0, 0, 0); + date_v2_data.push_back(*reinterpret_cast(&value)); + ColumnWithTypeAndName test_date(column->get_ptr(), std::make_shared(), + "test_date"); + + Block block({test_date}); + TypeDescriptor source_type(PrimitiveType::TYPE_DATEV2); + DateYearPartitionColumnTransform transform(source_type); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {47}; + std::vector expected_human_string = {"2017"}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + EXPECT_EQ(expected_human_string[i], + transform.to_human_string(transform.get_result_type(), result_data[i])); + } +} + +TEST_F(PartitionTransformersTest, test_timestamp_year_transform) { + auto column = ColumnDateTimeV2::create(); + auto& datetime_v2_data = column->get_data(); + DateV2Value value; + value.unchecked_set_time(2017, 11, 16, 22, 31, 8, 0); + datetime_v2_data.push_back(*reinterpret_cast(&value)); + ColumnWithTypeAndName test_timestamp(column->get_ptr(), std::make_shared(), + "test_timestamp"); + + Block block({test_timestamp}); + TypeDescriptor source_type(PrimitiveType::TYPE_DATETIMEV2); + TimestampYearPartitionColumnTransform transform(source_type); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {47}; + std::vector expected_human_string = {"2017"}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + EXPECT_EQ(expected_human_string[i], + transform.to_human_string(transform.get_result_type(), result_data[i])); + } +} + +TEST_F(PartitionTransformersTest, test_date_month_transform) { + auto column = ColumnDateV2::create(); + auto& date_v2_data = column->get_data(); + DateV2Value value; + value.unchecked_set_time(2017, 11, 16, 0, 0, 0, 0); + date_v2_data.push_back(*reinterpret_cast(&value)); + ColumnWithTypeAndName test_date(column->get_ptr(), std::make_shared(), + "test_date"); + + Block block({test_date}); + TypeDescriptor source_type(PrimitiveType::TYPE_DATEV2); + DateMonthPartitionColumnTransform transform(source_type); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {574}; + std::vector expected_human_string = {"2017-11"}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + EXPECT_EQ(expected_human_string[i], + transform.to_human_string(transform.get_result_type(), result_data[i])); + } +} + +TEST_F(PartitionTransformersTest, test_timestamp_month_transform) { + auto column = ColumnDateTimeV2::create(); + auto& datetime_v2_data = column->get_data(); + DateV2Value value; + value.unchecked_set_time(2017, 11, 16, 22, 31, 8, 0); + datetime_v2_data.push_back(*reinterpret_cast(&value)); + ColumnWithTypeAndName test_timestamp(column->get_ptr(), std::make_shared(), + "test_timestamp"); + + Block block({test_timestamp}); + TypeDescriptor source_type(PrimitiveType::TYPE_DATETIMEV2); + TimestampMonthPartitionColumnTransform transform(source_type); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {574}; + std::vector expected_human_string = {"2017-11"}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + EXPECT_EQ(expected_human_string[i], + transform.to_human_string(transform.get_result_type(), result_data[i])); + } +} + +TEST_F(PartitionTransformersTest, test_date_day_transform) { + auto column = ColumnDateV2::create(); + auto& date_v2_data = column->get_data(); + DateV2Value value; + value.unchecked_set_time(2017, 11, 16, 0, 0, 0, 0); + date_v2_data.push_back(*reinterpret_cast(&value)); + ColumnWithTypeAndName test_date(column->get_ptr(), std::make_shared(), + "test_date"); + + Block block({test_date}); + TypeDescriptor source_type(PrimitiveType::TYPE_DATEV2); + DateDayPartitionColumnTransform transform(source_type); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {17486}; + std::vector expected_human_string = {"2017-11-16"}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + EXPECT_EQ(expected_human_string[i], + transform.to_human_string(transform.get_result_type(), result_data[i])); + } +} + +TEST_F(PartitionTransformersTest, test_timestamp_day_transform) { + auto column = ColumnDateTimeV2::create(); + auto& datetime_v2_data = column->get_data(); + DateV2Value value; + value.unchecked_set_time(2017, 11, 16, 22, 31, 8, 0); + datetime_v2_data.push_back(*reinterpret_cast(&value)); + ColumnWithTypeAndName test_timestamp(column->get_ptr(), std::make_shared(), + "test_timestamp"); + + Block block({test_timestamp}); + TypeDescriptor source_type(PrimitiveType::TYPE_DATETIMEV2); + TimestampDayPartitionColumnTransform transform(source_type); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {17486}; + std::vector expected_human_string = {"2017-11-16"}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + EXPECT_EQ(expected_human_string[i], + transform.to_human_string(transform.get_result_type(), result_data[i])); + } +} + +TEST_F(PartitionTransformersTest, test_timestamp_hour_transform) { + auto column = ColumnDateTimeV2::create(); + auto& datetime_v2_data = column->get_data(); + DateV2Value value; + value.unchecked_set_time(2017, 11, 16, 22, 31, 8, 0); + datetime_v2_data.push_back(*reinterpret_cast(&value)); + ColumnWithTypeAndName test_timestamp(column->get_ptr(), std::make_shared(), + "test_timestamp"); + + Block block({test_timestamp}); + TypeDescriptor source_type(PrimitiveType::TYPE_DATETIMEV2); + TimestampHourPartitionColumnTransform transform(source_type); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {419686}; + std::vector expected_human_string = {"2017-11-16-22"}; + EXPECT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]); + EXPECT_EQ(expected_human_string[i], + transform.to_human_string(transform.get_result_type(), result_data[i])); + } +} + +TEST_F(PartitionTransformersTest, test_void_transform) { + const std::vector values({1, -1}); + auto column = ColumnInt32::create(); + column->insert_many_fix_len_data(reinterpret_cast(values.data()), values.size()); + ColumnWithTypeAndName test_int(column->get_ptr(), std::make_shared(), + "test_int"); + + Block block({test_int}); + TypeDescriptor source_type(PrimitiveType::TYPE_INT); + VoidPartitionColumnTransform transform(source_type); + + auto result = transform.apply(block, 0); + + const auto& result_null_map_data = + assert_cast(result.column.get())->get_null_map_data(); + + for (size_t i = 0; i < result_null_map_data.size(); ++i) { + EXPECT_EQ(1, result_null_map_data[i]); + } +} + +TEST_F(PartitionTransformersTest, test_nullable_column_integer_truncate_transform) { + const std::vector values({1, -1}); + auto column = ColumnNullable::create(ColumnInt32::create(), ColumnUInt8::create()); + column->insert_data(nullptr, 0); + column->insert_many_fix_len_data(reinterpret_cast(values.data()), values.size()); + ColumnWithTypeAndName test_int( + column->get_ptr(), + std::make_shared(std::make_shared()), "test_int"); + + Block block({test_int}); + TypeDescriptor source_type(PrimitiveType::TYPE_INT); + IntegerTruncatePartitionColumnTransform transform(source_type, 10); + + auto result = transform.apply(block, 0); + + std::vector expected_data = {0, -10}; + std::vector expected_human_string = {"0", "-10"}; + const auto* result_column = assert_cast(result.column.get()); + const auto& result_data = + assert_cast(result_column->get_nested_column_ptr().get()) + ->get_data(); + const auto& null_map_column = result_column->get_null_map_column(); + + EXPECT_EQ(1, null_map_column[0]); + EXPECT_EQ(0, null_map_column[1]); + EXPECT_EQ(0, null_map_column[2]); + + for (size_t i = 0, j = 0; i < result_column->size(); ++i) { + if (null_map_column[i] == 0) { + EXPECT_EQ(expected_data[j], result_data[i]); + EXPECT_EQ(expected_human_string[j], + transform.to_human_string(transform.get_result_type(), result_data[i])); + ++j; + } + } +} + +} // namespace doris::vectorized