Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions be/src/core/block/block.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1258,21 +1258,6 @@ std::unique_ptr<Block> Block::create_same_struct_block(size_t size, bool is_rese
return temp_block;
}

void Block::shrink_char_type_column_suffix_zero(const std::vector<size_t>& char_type_idx) {
for (auto idx : char_type_idx) {
if (idx < data.size()) {
auto& col_and_name = this->get_by_position(idx);
if (col_and_name.column->is_exclusive()) {
col_and_name.column->assert_mutable()->shrink_padding_chars();
} else {
auto mutable_col = std::move(*col_and_name.column).mutate();
mutable_col->shrink_padding_chars();
col_and_name.column = std::move(mutable_col);
}
}
}
}

size_t MutableBlock::allocated_bytes() const {
size_t res = 0;
for (const auto& col : _columns) {
Expand Down
3 changes: 0 additions & 3 deletions be/src/core/block/block.h
Original file line number Diff line number Diff line change
Expand Up @@ -413,9 +413,6 @@ class Block {
return res;
}

// for String type or Array<String> type
void shrink_char_type_column_suffix_zero(const std::vector<size_t>& char_type_idx);

void clear_column_mem_not_keep(const std::vector<bool>& column_keep_flags,
bool need_keep_first);

Expand Down
4 changes: 0 additions & 4 deletions be/src/core/column/column.h
Original file line number Diff line number Diff line change
Expand Up @@ -113,10 +113,6 @@ class IColumn : public COW<IColumn> {
return nullptr;
}

// shrink the end zeros for ColumnStr(also for who has it nested). so nest column will call it for all nested.
// for non-str col, will reach here(do nothing). only ColumnStr will really shrink itself.
virtual void shrink_padding_chars() {}

// Only used in ColumnVariant to handle lifecycle of variant. Other columns would do nothing.
virtual void finalize() {}

Expand Down
4 changes: 0 additions & 4 deletions be/src/core/column/column_array.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -121,10 +121,6 @@ ColumnArray::ColumnArray(SharedTag, ColumnPtr nested_column, ColumnPtr offsets_c
*static_cast<const IColumn::Ptr&>(offsets));
}

void ColumnArray::shrink_padding_chars() {
data->shrink_padding_chars();
}

std::string ColumnArray::get_name() const {
return "Array(" + get_data().get_name() + ")";
}
Expand Down
2 changes: 0 additions & 2 deletions be/src/core/column/column_array.h
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,6 @@ class ColumnArray final : public COWHelper<IColumn, ColumnArray> {
offsets->sanity_check();
}

void shrink_padding_chars() override;

/** On the index i there is an offset to the beginning of the i + 1 -th element. */
using ColumnOffsets = ColumnOffset64;

Expand Down
27 changes: 3 additions & 24 deletions be/src/core/column/column_dictionary.h
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ class ColumnDictI32 final : public COWHelper<IColumn, ColumnDictI32> {
_dict.initialize_hash_values_for_runtime_filter();
}

uint32_t get_hash_value(uint32_t idx) const { return _dict.get_hash_value(_codes[idx], _type); }
uint32_t get_hash_value(uint32_t idx) const { return _dict.get_hash_value(_codes[idx]); }

template <typename HybridSetType>
void find_codes(const HybridSetType* values, std::vector<UInt8>& selected) const {
Expand Down Expand Up @@ -278,14 +278,6 @@ class ColumnDictI32 final : public COWHelper<IColumn, ColumnDictI32> {

inline const StringRef& get_value(value_type code) const { return _dict.get_value(code); }

inline StringRef get_shrink_value(value_type code) const {
StringRef result = _dict.get_value(code);
if (_type == FieldType::OLAP_FIELD_TYPE_CHAR) {
result.size = strnlen(result.data, result.size);
}
return result;
}

size_t dict_size() const { return _dict.size(); }

std::string dict_debug_string() const { return _dict.debug_string(); }
Expand Down Expand Up @@ -326,26 +318,13 @@ class ColumnDictI32 final : public COWHelper<IColumn, ColumnDictI32> {
}
}

inline uint32_t get_hash_value(Int32 code, FieldType type) const {
Comment thread
csun5285 marked this conversation as resolved.
Resolved
inline uint32_t get_hash_value(Int32 code) const {
if (_compute_hash_value_flags[code]) {
return _hash_values[code];
} else {
auto& sv = (*_dict_data)[code];
// The char data is stored in the disk with the schema length,
// and zeros are filled if the length is insufficient

// When reading data, use shrink_char_type_column_suffix_zero(_char_type_idx)
// Remove the suffix 0
// When writing data, use the CharField::consume function to fill in the trailing 0.

// For dictionary data of char type, sv.size is the schema length,
// so use strnlen to remove the 0 at the end to get the actual length.
size_t len = sv.size;
if (type == FieldType::OLAP_FIELD_TYPE_CHAR) {
len = strnlen(sv.data, sv.size);
}
uint32_t hash_val =
crc32c::Extend(0, (const uint8_t*)sv.data, static_cast<uint32_t>(len));
crc32c::Extend(0, (const uint8_t*)sv.data, static_cast<uint32_t>(sv.size));
Comment thread
csun5285 marked this conversation as resolved.
Resolved
_hash_values[code] = hash_val;
_compute_hash_value_flags[code] = 1;
return _hash_values[code];
Expand Down
5 changes: 0 additions & 5 deletions be/src/core/column/column_map.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -700,11 +700,6 @@ Status ColumnMap::deduplicate_keys(bool recursive) {
return Status::OK();
}

void ColumnMap::shrink_padding_chars() {
keys_column->shrink_padding_chars();
values_column->shrink_padding_chars();
}

void ColumnMap::reserve(size_t n) {
get_offsets().reserve(n);
keys_column->reserve(n);
Expand Down
1 change: 0 additions & 1 deletion be/src/core/column/column_map.h
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@ class ColumnMap final : public COWHelper<IColumn, ColumnMap> {
const char* deserialize_and_insert_from_arena(const char* pos) override;

void update_hash_with_value(size_t n, SipHash& hash) const override;
void shrink_padding_chars() override;
ColumnPtr filter(const Filter& filt, ssize_t result_size_hint) const override;
size_t filter(const Filter& filter) override;
MutableColumnPtr permute(const Permutation& perm, size_t limit) const override;
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/column/column_nullable.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -122,10 +122,6 @@ void ColumnNullable::replace_columns(ColumnPtr nested_column, ColumnPtr null_map
check_const_only_in_top_level();
}

void ColumnNullable::shrink_padding_chars() {
get_nested_column_ptr()->shrink_padding_chars();
}

void ColumnNullable::update_xxHash_with_value(size_t start, size_t end, uint64_t& hash,
const uint8_t* __restrict null_data) const {
if (!has_null(start, end)) {
Expand Down
2 changes: 0 additions & 2 deletions be/src/core/column/column_nullable.h
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,6 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable> {
_nested_column->sanity_check();
}

void shrink_padding_chars() override;

bool is_variable_length() const override { return _nested_column->is_variable_length(); }

std::string get_name() const override { return "Nullable(" + _nested_column->get_name() + ")"; }
Expand Down
23 changes: 0 additions & 23 deletions be/src/core/column/column_string.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -85,29 +85,6 @@ MutableColumnPtr ColumnStr<T>::clone_resized(size_t to_size) const {
return res;
}

template <typename T>
void ColumnStr<T>::shrink_padding_chars() {
if (size() == 0) {
return;
}
char* data = reinterpret_cast<char*>(chars.data());
auto* offset = offsets.data();
size_t size = offsets.size();

// deal the 0-th element. no need to move.
auto next_start = offset[0];
offset[0] = static_cast<T>(strnlen(data, size_at(0)));
for (size_t i = 1; i < size; i++) {
// get the i-th length and whole move it to cover the last's trailing void
auto length = strnlen(data + next_start, offset[i] - next_start);
memmove(data + offset[i - 1], data + next_start, length);
// offset i will be changed. so save the old value for (i+1)-th to get its length.
next_start = offset[i];
offset[i] = offset[i - 1] + static_cast<T>(length);
}
chars.resize_fill(offsets.back()); // just call it to shrink memory here. no possible to expand.
}

// This method is only called by MutableBlock::merge_ignore_overflow
// by hash join operator to collect build data to avoid
// the total string length of a ColumnStr<uint32_t> column exceeds the 4G limit.
Expand Down
2 changes: 0 additions & 2 deletions be/src/core/column/column_string.h
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,6 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {

MutableColumnPtr clone_resized(size_t to_size) const override;

void shrink_padding_chars() override;

Field operator[](size_t n) const override;

void get(size_t n, Field& res) const override;
Expand Down
6 changes: 0 additions & 6 deletions be/src/core/column/column_struct.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -338,12 +338,6 @@ MutableColumnPtr ColumnStruct::permute(const Permutation& perm, size_t limit) co
return ColumnStruct::create(new_columns);
}

void ColumnStruct::shrink_padding_chars() {
for (auto& column : columns) {
column->shrink_padding_chars();
}
}

void ColumnStruct::reserve(size_t n) {
const size_t tuple_size = columns.size();
for (size_t i = 0; i < tuple_size; ++i) {
Expand Down
2 changes: 0 additions & 2 deletions be/src/core/column/column_struct.h
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,6 @@ class ColumnStruct final : public COWHelper<IColumn, ColumnStruct> {

int compare_at(size_t n, size_t m, const IColumn& rhs_, int nan_direction_hint) const override;

void shrink_padding_chars() override;

void reserve(size_t n) override;
void resize(size_t n) override;
size_t byte_size() const override;
Expand Down
6 changes: 1 addition & 5 deletions be/src/core/column/predicate_column.h
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,7 @@ class PredicateColumnType final : public COWHelper<IColumn, PredicateColumnType<

StringRef get_data_at(size_t n) const override {
if constexpr (std::is_same_v<T, StringRef>) {
auto res = reinterpret_cast<const StringRef&>(data[n]);
Comment thread
csun5285 marked this conversation as resolved.
Resolved
if constexpr (Type == TYPE_CHAR) {
res.size = strnlen(res.data, res.size);
}
return res;
return reinterpret_cast<const StringRef&>(data[n]);
} else {
throw doris::Exception(
ErrorCode::INTERNAL_ERROR,
Expand Down
23 changes: 7 additions & 16 deletions be/src/core/data_type_serde/data_type_string_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -462,25 +462,16 @@ Status DataTypeStringSerDeBase<ColumnType>::from_string(StringRef& str, IColumn&

// Deserializes a STRING/VARCHAR/CHAR value from its OLAP string representation
// (e.g. from ZoneMap protobuf). This is the inverse of to_olap_string().
//
// For CHAR type: if the string is shorter than the declared column length (_len),
// pads with '\0' bytes to reach _len. This preserves CHAR's fixed-length semantics.
// For STRING/VARCHAR: stores the string as-is.
//
// Examples:
// CHAR(10), str="hello" => field = "hello\0\0\0\0\0" (10 bytes)
// VARCHAR, str="hello" => field = "hello" (5 bytes)
template <typename ColumnType>
Status DataTypeStringSerDeBase<ColumnType>::from_olap_string(const std::string& str, Field& field,
Comment thread
csun5285 marked this conversation as resolved.
Resolved
const FormatOptions& options) const {
if (cast_set<int>(str.size()) < _len) {
DCHECK_EQ(_type, TYPE_CHAR);
std::string tmp(_len, '\0');
memcpy(tmp.data(), str.data(), str.size());
field = Field::create_field<TYPE_CHAR>(std::move(tmp));
} else {
field = Field::create_field<TYPE_STRING>(str);
}
// CHAR(N) writes through OlapColumnDataConvertorChar are zero-padded to
// the declared schema length, so the serialized OLAP string carries
// trailing '\0' bytes. strnlen() drops that padding to surface the
// logical character content in the Field. VARCHAR / STRING never write
// trailing '\0' through this path, so strnlen is a no-op for them.
size_t len = strnlen(str.data(), str.size());
field = Field::create_field<TYPE_STRING>(std::string(str.data(), len));
Comment thread
csun5285 marked this conversation as resolved.
Resolved
return Status::OK();
}

Expand Down
46 changes: 0 additions & 46 deletions be/src/exec/rowid_fetcher.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -208,30 +208,6 @@ Status RowIDFetcher::_merge_rpc_results(const PMultiGetRequest& request,
return Status::OK();
}

bool _has_char_type(const DataTypePtr& type) {
switch (type->get_primitive_type()) {
case TYPE_CHAR: {
return true;
}
case TYPE_ARRAY: {
const auto* arr_type = assert_cast<const DataTypeArray*>(remove_nullable(type).get());
return _has_char_type(arr_type->get_nested_type());
}
case TYPE_MAP: {
const auto* map_type = assert_cast<const DataTypeMap*>(remove_nullable(type).get());
return _has_char_type(map_type->get_key_type()) ||
_has_char_type(map_type->get_value_type());
}
case TYPE_STRUCT: {
const auto* struct_type = assert_cast<const DataTypeStruct*>(remove_nullable(type).get());
return std::any_of(struct_type->get_elements().begin(), struct_type->get_elements().end(),
[&](const DataTypePtr& dt) -> bool { return _has_char_type(dt); });
}
default:
return false;
}
}

Status RowIDFetcher::fetch(const ColumnPtr& column_row_ids, Block* res_block) {
CHECK(!_stubs.empty());
PMultiGetRequest mget_req = _init_fetch_request(
Expand Down Expand Up @@ -281,16 +257,6 @@ Status RowIDFetcher::fetch(const ColumnPtr& column_row_ids, Block* res_block) {
}
// Check row consistency
RETURN_IF_CATCH_EXCEPTION(res_block->check_number_of_rows());
// shrink for char type
std::vector<size_t> char_type_idx;
for (size_t i = 0; i < _fetch_option.desc->slots().size(); i++) {
const auto& column_desc = _fetch_option.desc->slots()[i];
const auto type = column_desc->type();
if (_has_char_type(type)) {
char_type_idx.push_back(i);
}
}
res_block->shrink_char_type_column_suffix_zero(char_type_idx);
VLOG_DEBUG << "dump block:" << res_block->dump_data(0, 10);
return Status::OK();
}
Expand Down Expand Up @@ -595,15 +561,6 @@ Status RowIdStorageReader::read_by_rowids(const PMultiGetRequestV2& request,
for (const auto& pslot : request_block_desc.slots()) {
slots.push_back(SlotDescriptor(pslot));
}
// prepare block char vector shrink for char type
std::vector<size_t> char_type_idx;
for (int j = 0; j < slots.size(); ++j) {
auto slot = slots[j];
if (_has_char_type(slot.type())) {
char_type_idx.push_back(j);
}
}

try {
if (first_file_mapping->type == FileMappingType::INTERNAL) {
RETURN_IF_ERROR(read_batch_doris_format_row(
Expand All @@ -621,9 +578,6 @@ Status RowIdStorageReader::read_by_rowids(const PMultiGetRequestV2& request,
return Status::Error<false>(e.code(), "Row id fetch failed because {}",
e.what());
}

// after read the block, shrink char type block
result_blocks[i].shrink_char_type_column_suffix_zero(char_type_idx);
}

[[maybe_unused]] size_t compressed_size = 0;
Expand Down
14 changes: 1 addition & 13 deletions be/src/exprs/bloom_filter_func_adaptor.h
Original file line number Diff line number Diff line change
Expand Up @@ -243,18 +243,6 @@ struct StringFindOp : CommonFindOp<fixed_len_to_uint32_method, StringRef> {
}
};

// We do not need to judge whether data is empty, because null will not appear
// when filer used by the storage engine
template <typename fixed_len_to_uint32_method>
struct FixedStringFindOp : public StringFindOp<fixed_len_to_uint32_method> {
static uint16_t find_batch_olap_engine(const BloomFilterAdaptor& bloom_filter, const char* data,
const uint8_t* nullmap, uint16_t* offsets, int number,
const bool is_parse_column) {
return find_batch_olap<fixed_len_to_uint32_method, StringRef, true>(
bloom_filter, data, nullmap, offsets, number, is_parse_column);
}
};

template <typename fixed_len_to_uint32_method, PrimitiveType type>
struct BloomFilterTypeTraits {
using T = typename PrimitiveTypeTraits<type>::CppType;
Expand All @@ -263,7 +251,7 @@ struct BloomFilterTypeTraits {

template <typename fixed_len_to_uint32_method>
struct BloomFilterTypeTraits<fixed_len_to_uint32_method, TYPE_CHAR> {
using FindOp = FixedStringFindOp<fixed_len_to_uint32_method>;
using FindOp = StringFindOp<fixed_len_to_uint32_method>;
};

template <typename fixed_len_to_uint32_method>
Expand Down
15 changes: 2 additions & 13 deletions be/src/exprs/bloom_filter_func_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -54,23 +54,12 @@ struct fixed_len_to_uint32_v2 {
}
};

template <typename fixed_len_to_uint32_method, typename T, bool need_trim = false>
template <typename fixed_len_to_uint32_method, typename T>
uint16_t find_batch_olap(const BloomFilterAdaptor& bloom_filter, const char* data,
const uint8_t* nullmap, uint16_t* offsets, int number,
const bool is_parse_column) {
auto get_element = [](const char* input_data, int idx) {
if constexpr (std::is_same_v<T, StringRef> && need_trim) {
const auto value = ((const StringRef*)(input_data))[idx];
int64_t size = value.size;
const char* data = value.data;
// CHAR type may pad the tail with \0, need to trim
while (size > 0 && data[size - 1] == '\0') {
size--;
}
return StringRef(value.data, size);
} else {
return ((const T*)(input_data))[idx];
}
return ((const T*)(input_data))[idx];
};

uint16_t new_size = 0;
Expand Down
Loading
Loading