diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 5a53acbeacee94..ee4ea8bf092be3 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1626,6 +1626,8 @@ DEFINE_Validator(binary_plain_encoding_default_impl, [](const std::string& confi return config == "v1" || config == "v2"; }); +DEFINE_mBool(integer_type_default_use_plain_encoding, "true"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index 3297bfea8ebe95..8489554394a534 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1666,6 +1666,8 @@ DECLARE_mString(aws_credentials_provider_version); DECLARE_mString(binary_plain_encoding_default_impl); +DECLARE_mBool(integer_type_default_use_plain_encoding); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/olap/rowset/segment_v2/encoding_info.cpp b/be/src/olap/rowset/segment_v2/encoding_info.cpp index 6fcf571c038071..31216c0e3fafff 100644 --- a/be/src/olap/rowset/segment_v2/encoding_info.cpp +++ b/be/src/olap/rowset/segment_v2/encoding_info.cpp @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -317,6 +318,80 @@ EncodingInfoResolver::~EncodingInfoResolver() { _encoding_map.clear(); } +namespace { +bool is_integer_type(FieldType type) { + return type == FieldType::OLAP_FIELD_TYPE_TINYINT || + type == FieldType::OLAP_FIELD_TYPE_SMALLINT || type == FieldType::OLAP_FIELD_TYPE_INT || + type == FieldType::OLAP_FIELD_TYPE_BIGINT || type == FieldType::OLAP_FIELD_TYPE_LARGEINT; +} + +bool is_binary_type(FieldType type) { + return type == FieldType::OLAP_FIELD_TYPE_CHAR || type == FieldType::OLAP_FIELD_TYPE_VARCHAR || + type == FieldType::OLAP_FIELD_TYPE_STRING || type == FieldType::OLAP_FIELD_TYPE_JSONB || + type == FieldType::OLAP_FIELD_TYPE_VARIANT || type == FieldType::OLAP_FIELD_TYPE_HLL || + type == FieldType::OLAP_FIELD_TYPE_BITMAP || + type == FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE || + type == FieldType::OLAP_FIELD_TYPE_AGG_STATE; +} +} // namespace + +EncodingTypePB EncodingInfoResolver::get_default_encoding(FieldType type, + bool optimize_value_seek) const { + // Predicate for default encoding transformation + // Parameters: (type, current_default_encoding, optimize_value_seek) + // Returns: true if the transformation should be applied + using Predicate = std::function; + + // Hook for transforming default encoding: predicate -> target encoding + struct EncodingTransform { + Predicate predicate; + EncodingTypePB target_encoding; + }; + + // Static array of hooks for default encoding transformations + static const std::vector hooks = { + // Hook 1: Binary types - PLAIN_ENCODING -> PLAIN_ENCODING_V2 + // Applies when: type is binary, encoding is PLAIN_ENCODING, and config enables v2 + EncodingTransform { + .predicate = + [](FieldType type, EncodingTypePB encoding, bool optimize_value_seek) { + return encoding == PLAIN_ENCODING && is_binary_type(type) && + config::binary_plain_encoding_default_impl == "v2"; + }, + .target_encoding = PLAIN_ENCODING_V2}, + + // Hook 2: Integer types - any encoding -> PLAIN_ENCODING + // Applies when: type is integer and config enables plain encoding for integers + EncodingTransform { + .predicate = + [](FieldType type, EncodingTypePB encoding, bool optimize_value_seek) { + return is_integer_type(type) && + config::integer_type_default_use_plain_encoding; + }, + .target_encoding = PLAIN_ENCODING}}; + + auto& encoding_map = + optimize_value_seek ? _value_seek_encoding_map : _default_encoding_type_map; + auto it = encoding_map.find(type); + if (it != encoding_map.end()) { + EncodingTypePB encoding = it->second; + + // Apply hooks in order to transform the default encoding + for (const auto& hook : hooks) { + if (hook.predicate(type, encoding, optimize_value_seek)) { + // Verify target encoding is available for this type + if (_encoding_map.contains(std::make_pair(type, hook.target_encoding))) { + encoding = hook.target_encoding; + break; // Apply only the first matching hook + } + } + } + + return encoding; + } + return UNKNOWN_ENCODING; +} + Status EncodingInfoResolver::get(FieldType data_type, EncodingTypePB encoding_type, const EncodingInfo** out) { if (encoding_type == DEFAULT_ENCODING) { diff --git a/be/src/olap/rowset/segment_v2/encoding_info.h b/be/src/olap/rowset/segment_v2/encoding_info.h index 92d87459398fbf..46d0f3f7581b9e 100644 --- a/be/src/olap/rowset/segment_v2/encoding_info.h +++ b/be/src/olap/rowset/segment_v2/encoding_info.h @@ -103,21 +103,7 @@ class EncodingInfoResolver { EncodingInfoResolver(); ~EncodingInfoResolver(); - EncodingTypePB get_default_encoding(FieldType type, bool optimize_value_seek) const { - auto& encoding_map = - optimize_value_seek ? _value_seek_encoding_map : _default_encoding_type_map; - auto it = encoding_map.find(type); - if (it != encoding_map.end()) { - EncodingTypePB encoding = it->second; - // For binary types, use PLAIN_ENCODING_V2 if config::binary_plain_encoding_default_impl is "v2" - if (encoding == PLAIN_ENCODING && config::binary_plain_encoding_default_impl == "v2" && - _encoding_map.contains(std::make_pair(type, PLAIN_ENCODING_V2))) { - return PLAIN_ENCODING_V2; - } - return encoding; - } - return UNKNOWN_ENCODING; - } + EncodingTypePB get_default_encoding(FieldType type, bool optimize_value_seek) const; Status get(FieldType data_type, EncodingTypePB encoding_type, const EncodingInfo** out); diff --git a/be/src/olap/rowset/segment_v2/plain_page.h b/be/src/olap/rowset/segment_v2/plain_page.h index 1a537e5b8b7912..2065a5a899112a 100644 --- a/be/src/olap/rowset/segment_v2/plain_page.h +++ b/be/src/olap/rowset/segment_v2/plain_page.h @@ -25,6 +25,7 @@ #include "olap/types.h" #include "util/coding.h" #include "util/faststring.h" +#include "vec/common/unaligned.h" namespace doris { #include "common/compile_check_begin.h" @@ -33,7 +34,7 @@ namespace segment_v2 { static const size_t PLAIN_PAGE_HEADER_SIZE = sizeof(uint32_t); template -class PlainPageBuilder : public PageBuilderHelper > { +class PlainPageBuilder : public PageBuilderHelper> { public: using Self = PlainPageBuilder; friend class PageBuilderHelper; @@ -41,24 +42,28 @@ class PlainPageBuilder : public PageBuilderHelper > { Status init() override { // Reserve enough space for the page, plus a bit of slop since // we often overrun the page by a few values. - RETURN_IF_CATCH_EXCEPTION(_buffer.reserve(_options.data_page_size + 1024)); + RETURN_IF_CATCH_EXCEPTION(_buffer.reserve(_options.data_page_size)); return reset(); } - bool is_page_full() override { return _buffer.size() > _options.data_page_size; } + bool is_page_full() override { return _remain_element_capacity == 0; } Status add(const uint8_t* vals, size_t* count) override { - if (is_page_full()) { + if (is_page_full() || *count == 0) { *count = 0; return Status::OK(); } size_t old_size = _buffer.size(); + size_t to_add = std::min(_remain_element_capacity, *count); // This may need a large memory, should return error if could not allocated // successfully, to avoid BE OOM. - RETURN_IF_CATCH_EXCEPTION(_buffer.resize(old_size + *count * SIZE_OF_TYPE)); - memcpy(&_buffer[old_size], vals, *count * SIZE_OF_TYPE); - _count += *count; - _raw_data_size += *count * SIZE_OF_TYPE; + RETURN_IF_CATCH_EXCEPTION(_buffer.resize(old_size + to_add * SIZE_OF_TYPE)); + memcpy(&_buffer[old_size], vals, to_add * SIZE_OF_TYPE); + _count += to_add; + _raw_data_size += to_add * SIZE_OF_TYPE; + + *count = to_add; + _remain_element_capacity -= to_add; return Status::OK(); } @@ -78,11 +83,12 @@ class PlainPageBuilder : public PageBuilderHelper > { Status reset() override { RETURN_IF_CATCH_EXCEPTION({ - _buffer.reserve(_options.data_page_size + 1024); + _buffer.reserve(_options.data_page_size); _count = 0; _raw_data_size = 0; _buffer.clear(); _buffer.resize(PLAIN_PAGE_HEADER_SIZE); + _remain_element_capacity = _options.data_page_size / SIZE_OF_TYPE; }); return Status::OK(); } @@ -115,6 +121,7 @@ class PlainPageBuilder : public PageBuilderHelper > { faststring _buffer; PageBuilderOptions _options; size_t _count; + size_t _remain_element_capacity {0}; uint64_t _raw_data_size = 0; typedef typename TypeTraits::CppType CppType; enum { SIZE_OF_TYPE = TypeTraits::size }; @@ -204,7 +211,48 @@ class PlainPageDecoder : public PageDecoder { } Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst) override { - return Status::NotSupported("plain page not implement vec op now"); + DCHECK(_parsed); + if (*n == 0 || _cur_idx >= _num_elems) [[unlikely]] { + return Status::OK(); + } + + size_t max_fetch = std::min(*n, static_cast(_num_elems - _cur_idx)); + const void* src_data = &_data[PLAIN_PAGE_HEADER_SIZE + _cur_idx * SIZE_OF_TYPE]; + + dst->insert_many_fix_len_data((const char*)src_data, max_fetch); + + *n = max_fetch; + _cur_idx += max_fetch; + + return Status::OK(); + } + + Status read_by_rowids(const rowid_t* rowids, ordinal_t page_first_ordinal, size_t* n, + vectorized::MutableColumnPtr& dst) override { + DCHECK(_parsed); + if (*n == 0) [[unlikely]] { + return Status::OK(); + } + + auto total = *n; + auto read_count = 0; + _buffer.resize(total); + for (size_t i = 0; i < total; ++i) { + ordinal_t ord = rowids[i] - page_first_ordinal; + if (UNLIKELY(ord >= _num_elems)) { + break; + } + + _buffer[read_count++] = + unaligned_load(&_data[PLAIN_PAGE_HEADER_SIZE + ord * SIZE_OF_TYPE]); + } + + if (LIKELY(read_count > 0)) { + dst->insert_many_fix_len_data((char*)_buffer.data(), read_count); + } + + *n = read_count; + return Status::OK(); } size_t count() const override { @@ -225,6 +273,8 @@ class PlainPageDecoder : public PageDecoder { uint32_t _cur_idx; typedef typename TypeTraits::CppType CppType; enum { SIZE_OF_TYPE = TypeTraits::size }; + + std::vector, uint8_t, CppType>> _buffer; }; } // namespace segment_v2 diff --git a/be/test/olap/rowset/segment_v2/encoding_info_test.cpp b/be/test/olap/rowset/segment_v2/encoding_info_test.cpp index ab0361cc270a2d..365722df4a4b60 100644 --- a/be/test/olap/rowset/segment_v2/encoding_info_test.cpp +++ b/be/test/olap/rowset/segment_v2/encoding_info_test.cpp @@ -109,13 +109,15 @@ TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { const auto* bigint_type_info = get_scalar_type_info(); config::binary_plain_encoding_default_impl = "v1"; + auto expected_encoding = + (config::integer_type_default_use_plain_encoding) ? PLAIN_ENCODING : BIT_SHUFFLE; EncodingTypePB encoding_type = EncodingInfo::get_default_encoding(bigint_type_info->type(), false); - EXPECT_EQ(BIT_SHUFFLE, encoding_type); + EXPECT_EQ(expected_encoding, encoding_type); config::binary_plain_encoding_default_impl = "v2"; encoding_type = EncodingInfo::get_default_encoding(bigint_type_info->type(), false); - EXPECT_EQ(BIT_SHUFFLE, encoding_type); // Should remain BIT_SHUFFLE + EXPECT_EQ(expected_encoding, encoding_type); // Should remain BIT_SHUFFLE // Reset to default config::binary_plain_encoding_default_impl = "v1"; diff --git a/regression-test/suites/show_p0/test_show_data_warehouse.groovy b/regression-test/suites/show_p0/test_show_data_warehouse.groovy index 55388b17d8a4a3..8dadd0bb61a4db 100644 --- a/regression-test/suites/show_p0/test_show_data_warehouse.groovy +++ b/regression-test/suites/show_p0/test_show_data_warehouse.groovy @@ -88,17 +88,19 @@ suite("test_show_data_warehouse") { boolean hitDb1 = false; boolean hitDb2 = false; - long db1Size = 945 * replicaCount1 - long db2Size = 922 * replicaCount1 + long db1Size = 988 * replicaCount1 + long db2Size = 875 * replicaCount2 def result; do { current = System.currentTimeMillis() result = sql """ show data properties("entire_warehouse"="true","db_names"="${db1Name}"); """ + log.info("show data warehouse db1 result: ${result}"); if ((result.size() == 2) && result[0][1].toInteger() >= db1Size) { hitDb1 = true; } result = sql """ show data properties("entire_warehouse"="true","db_names"="${db2Name}"); """ + log.info("show data warehouse db2 result: ${result}"); if (result.size() == 2 && result[0][1].toInteger() >= db2Size) { hitDb2 = true; } @@ -112,6 +114,7 @@ suite("test_show_data_warehouse") { assertTrue((hitDb1 && hitDb2), "data size check fail after 1200s") result = sql """ show data properties("entire_warehouse"="true","db_names"="${db1Name},${db2Name}"); """ + log.info("show data warehouse db1 and db2 result: ${result}"); assertEquals(result.size(), 3) assert result[0][1].toInteger() >= db1Size assert result[1][1].toInteger() >= db2Size