Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions be/src/common/config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1626,6 +1626,8 @@ DEFINE_Validator(binary_plain_encoding_default_impl, [](const std::string& confi
return config == "v1" || config == "v2";
});

DEFINE_mBool(integer_type_default_use_plain_encoding, "true");

// clang-format off
#ifdef BE_TEST
// test s3
Expand Down
2 changes: 2 additions & 0 deletions be/src/common/config.h
Original file line number Diff line number Diff line change
Expand Up @@ -1666,6 +1666,8 @@ DECLARE_mString(aws_credentials_provider_version);

DECLARE_mString(binary_plain_encoding_default_impl);

DECLARE_mBool(integer_type_default_use_plain_encoding);

#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
Expand Down
75 changes: 75 additions & 0 deletions be/src/olap/rowset/segment_v2/encoding_info.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include <gen_cpp/segment_v2.pb.h>

#include <array>
#include <iterator>
#include <type_traits>
#include <unordered_map>
Expand Down Expand Up @@ -317,6 +318,80 @@ EncodingInfoResolver::~EncodingInfoResolver() {
_encoding_map.clear();
}

namespace {
bool is_integer_type(FieldType type) {
return type == FieldType::OLAP_FIELD_TYPE_TINYINT ||
type == FieldType::OLAP_FIELD_TYPE_SMALLINT || type == FieldType::OLAP_FIELD_TYPE_INT ||
type == FieldType::OLAP_FIELD_TYPE_BIGINT || type == FieldType::OLAP_FIELD_TYPE_LARGEINT;
}

bool is_binary_type(FieldType type) {
return type == FieldType::OLAP_FIELD_TYPE_CHAR || type == FieldType::OLAP_FIELD_TYPE_VARCHAR ||
type == FieldType::OLAP_FIELD_TYPE_STRING || type == FieldType::OLAP_FIELD_TYPE_JSONB ||
type == FieldType::OLAP_FIELD_TYPE_VARIANT || type == FieldType::OLAP_FIELD_TYPE_HLL ||
type == FieldType::OLAP_FIELD_TYPE_BITMAP ||
type == FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE ||
type == FieldType::OLAP_FIELD_TYPE_AGG_STATE;
}
} // namespace

EncodingTypePB EncodingInfoResolver::get_default_encoding(FieldType type,
bool optimize_value_seek) const {
// Predicate for default encoding transformation
// Parameters: (type, current_default_encoding, optimize_value_seek)
// Returns: true if the transformation should be applied
using Predicate = std::function<bool(FieldType, EncodingTypePB, bool)>;

// Hook for transforming default encoding: predicate -> target encoding
struct EncodingTransform {
Predicate predicate;
EncodingTypePB target_encoding;
};

// Static array of hooks for default encoding transformations
static const std::vector<EncodingTransform> hooks = {
// Hook 1: Binary types - PLAIN_ENCODING -> PLAIN_ENCODING_V2
// Applies when: type is binary, encoding is PLAIN_ENCODING, and config enables v2
EncodingTransform {
.predicate =
[](FieldType type, EncodingTypePB encoding, bool optimize_value_seek) {
return encoding == PLAIN_ENCODING && is_binary_type(type) &&
config::binary_plain_encoding_default_impl == "v2";
},
.target_encoding = PLAIN_ENCODING_V2},

// Hook 2: Integer types - any encoding -> PLAIN_ENCODING
// Applies when: type is integer and config enables plain encoding for integers
EncodingTransform {
.predicate =
[](FieldType type, EncodingTypePB encoding, bool optimize_value_seek) {
return is_integer_type(type) &&
config::integer_type_default_use_plain_encoding;
},
.target_encoding = PLAIN_ENCODING}};

auto& encoding_map =
optimize_value_seek ? _value_seek_encoding_map : _default_encoding_type_map;
auto it = encoding_map.find(type);
if (it != encoding_map.end()) {
EncodingTypePB encoding = it->second;

// Apply hooks in order to transform the default encoding
for (const auto& hook : hooks) {
if (hook.predicate(type, encoding, optimize_value_seek)) {
// Verify target encoding is available for this type
if (_encoding_map.contains(std::make_pair(type, hook.target_encoding))) {
encoding = hook.target_encoding;
break; // Apply only the first matching hook
}
}
}

return encoding;
}
return UNKNOWN_ENCODING;
}

Status EncodingInfoResolver::get(FieldType data_type, EncodingTypePB encoding_type,
const EncodingInfo** out) {
if (encoding_type == DEFAULT_ENCODING) {
Expand Down
16 changes: 1 addition & 15 deletions be/src/olap/rowset/segment_v2/encoding_info.h
Original file line number Diff line number Diff line change
Expand Up @@ -103,21 +103,7 @@ class EncodingInfoResolver {
EncodingInfoResolver();
~EncodingInfoResolver();

EncodingTypePB get_default_encoding(FieldType type, bool optimize_value_seek) const {
auto& encoding_map =
optimize_value_seek ? _value_seek_encoding_map : _default_encoding_type_map;
auto it = encoding_map.find(type);
if (it != encoding_map.end()) {
EncodingTypePB encoding = it->second;
// For binary types, use PLAIN_ENCODING_V2 if config::binary_plain_encoding_default_impl is "v2"
if (encoding == PLAIN_ENCODING && config::binary_plain_encoding_default_impl == "v2" &&
_encoding_map.contains(std::make_pair(type, PLAIN_ENCODING_V2))) {
return PLAIN_ENCODING_V2;
}
return encoding;
}
return UNKNOWN_ENCODING;
}
EncodingTypePB get_default_encoding(FieldType type, bool optimize_value_seek) const;

Status get(FieldType data_type, EncodingTypePB encoding_type, const EncodingInfo** out);

Expand Down
70 changes: 60 additions & 10 deletions be/src/olap/rowset/segment_v2/plain_page.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include "olap/types.h"
#include "util/coding.h"
#include "util/faststring.h"
#include "vec/common/unaligned.h"

namespace doris {
#include "common/compile_check_begin.h"
Expand All @@ -33,32 +34,36 @@ namespace segment_v2 {
static const size_t PLAIN_PAGE_HEADER_SIZE = sizeof(uint32_t);

template <FieldType Type>
class PlainPageBuilder : public PageBuilderHelper<PlainPageBuilder<Type> > {
class PlainPageBuilder : public PageBuilderHelper<PlainPageBuilder<Type>> {
public:
using Self = PlainPageBuilder<Type>;
friend class PageBuilderHelper<Self>;

Status init() override {
// Reserve enough space for the page, plus a bit of slop since
// we often overrun the page by a few values.
RETURN_IF_CATCH_EXCEPTION(_buffer.reserve(_options.data_page_size + 1024));
RETURN_IF_CATCH_EXCEPTION(_buffer.reserve(_options.data_page_size));
return reset();
}

bool is_page_full() override { return _buffer.size() > _options.data_page_size; }
bool is_page_full() override { return _remain_element_capacity == 0; }

Status add(const uint8_t* vals, size_t* count) override {
if (is_page_full()) {
if (is_page_full() || *count == 0) {
*count = 0;
return Status::OK();
}
size_t old_size = _buffer.size();
size_t to_add = std::min(_remain_element_capacity, *count);
// This may need a large memory, should return error if could not allocated
// successfully, to avoid BE OOM.
RETURN_IF_CATCH_EXCEPTION(_buffer.resize(old_size + *count * SIZE_OF_TYPE));
memcpy(&_buffer[old_size], vals, *count * SIZE_OF_TYPE);
_count += *count;
_raw_data_size += *count * SIZE_OF_TYPE;
RETURN_IF_CATCH_EXCEPTION(_buffer.resize(old_size + to_add * SIZE_OF_TYPE));
memcpy(&_buffer[old_size], vals, to_add * SIZE_OF_TYPE);
_count += to_add;
_raw_data_size += to_add * SIZE_OF_TYPE;

*count = to_add;
_remain_element_capacity -= to_add;
return Status::OK();
}

Expand All @@ -78,11 +83,12 @@ class PlainPageBuilder : public PageBuilderHelper<PlainPageBuilder<Type> > {

Status reset() override {
RETURN_IF_CATCH_EXCEPTION({
_buffer.reserve(_options.data_page_size + 1024);
_buffer.reserve(_options.data_page_size);
_count = 0;
_raw_data_size = 0;
_buffer.clear();
_buffer.resize(PLAIN_PAGE_HEADER_SIZE);
_remain_element_capacity = _options.data_page_size / SIZE_OF_TYPE;
});
return Status::OK();
}
Expand Down Expand Up @@ -115,6 +121,7 @@ class PlainPageBuilder : public PageBuilderHelper<PlainPageBuilder<Type> > {
faststring _buffer;
PageBuilderOptions _options;
size_t _count;
size_t _remain_element_capacity {0};
uint64_t _raw_data_size = 0;
typedef typename TypeTraits<Type>::CppType CppType;
enum { SIZE_OF_TYPE = TypeTraits<Type>::size };
Expand Down Expand Up @@ -204,7 +211,48 @@ class PlainPageDecoder : public PageDecoder {
}

Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst) override {
return Status::NotSupported("plain page not implement vec op now");
DCHECK(_parsed);
if (*n == 0 || _cur_idx >= _num_elems) [[unlikely]] {
return Status::OK();
}

size_t max_fetch = std::min(*n, static_cast<size_t>(_num_elems - _cur_idx));
const void* src_data = &_data[PLAIN_PAGE_HEADER_SIZE + _cur_idx * SIZE_OF_TYPE];

dst->insert_many_fix_len_data((const char*)src_data, max_fetch);

*n = max_fetch;
_cur_idx += max_fetch;

return Status::OK();
}

Status read_by_rowids(const rowid_t* rowids, ordinal_t page_first_ordinal, size_t* n,
vectorized::MutableColumnPtr& dst) override {
DCHECK(_parsed);
if (*n == 0) [[unlikely]] {
return Status::OK();
}

auto total = *n;
auto read_count = 0;
_buffer.resize(total);
for (size_t i = 0; i < total; ++i) {
ordinal_t ord = rowids[i] - page_first_ordinal;
if (UNLIKELY(ord >= _num_elems)) {
break;
}

_buffer[read_count++] =
unaligned_load<CppType>(&_data[PLAIN_PAGE_HEADER_SIZE + ord * SIZE_OF_TYPE]);
}

if (LIKELY(read_count > 0)) {
dst->insert_many_fix_len_data((char*)_buffer.data(), read_count);
}

*n = read_count;
return Status::OK();
}

size_t count() const override {
Expand All @@ -225,6 +273,8 @@ class PlainPageDecoder : public PageDecoder {
uint32_t _cur_idx;
typedef typename TypeTraits<Type>::CppType CppType;
enum { SIZE_OF_TYPE = TypeTraits<Type>::size };

std::vector<std::conditional_t<std::is_same_v<CppType, bool>, uint8_t, CppType>> _buffer;
};

} // namespace segment_v2
Expand Down
6 changes: 4 additions & 2 deletions be/test/olap/rowset/segment_v2/encoding_info_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -109,13 +109,15 @@ TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) {
const auto* bigint_type_info = get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_BIGINT>();

config::binary_plain_encoding_default_impl = "v1";
auto expected_encoding =
(config::integer_type_default_use_plain_encoding) ? PLAIN_ENCODING : BIT_SHUFFLE;
EncodingTypePB encoding_type =
EncodingInfo::get_default_encoding(bigint_type_info->type(), false);
EXPECT_EQ(BIT_SHUFFLE, encoding_type);
EXPECT_EQ(expected_encoding, encoding_type);

config::binary_plain_encoding_default_impl = "v2";
encoding_type = EncodingInfo::get_default_encoding(bigint_type_info->type(), false);
EXPECT_EQ(BIT_SHUFFLE, encoding_type); // Should remain BIT_SHUFFLE
EXPECT_EQ(expected_encoding, encoding_type); // Should remain BIT_SHUFFLE

// Reset to default
config::binary_plain_encoding_default_impl = "v1";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,17 +88,19 @@ suite("test_show_data_warehouse") {
boolean hitDb1 = false;
boolean hitDb2 = false;

long db1Size = 945 * replicaCount1
long db2Size = 922 * replicaCount1
long db1Size = 988 * replicaCount1
long db2Size = 875 * replicaCount2
def result;
do {
current = System.currentTimeMillis()
result = sql """ show data properties("entire_warehouse"="true","db_names"="${db1Name}"); """
log.info("show data warehouse db1 result: ${result}");
if ((result.size() == 2) && result[0][1].toInteger() >= db1Size) {
hitDb1 = true;
}

result = sql """ show data properties("entire_warehouse"="true","db_names"="${db2Name}"); """
log.info("show data warehouse db2 result: ${result}");
if (result.size() == 2 && result[0][1].toInteger() >= db2Size) {
hitDb2 = true;
}
Expand All @@ -112,6 +114,7 @@ suite("test_show_data_warehouse") {
assertTrue((hitDb1 && hitDb2), "data size check fail after 1200s")

result = sql """ show data properties("entire_warehouse"="true","db_names"="${db1Name},${db2Name}"); """
log.info("show data warehouse db1 and db2 result: ${result}");
assertEquals(result.size(), 3)
assert result[0][1].toInteger() >= db1Size
assert result[1][1].toInteger() >= db2Size
Expand Down