Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 135 additions & 12 deletions be/src/format/table/equality_delete.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,111 @@

#include "format/table/equality_delete.h"

#include "core/column/column_const.h"
#include "core/column/column_nullable.h"
#include "exprs/create_predicate_function.h"
#include "util/hash_util.hpp"

namespace doris {

namespace {

// Iceberg BINARY / FIXED keys are read as VARBINARY and string keys as STRING, and the delete file and
// the data file may carry the same key in different ones of these classes (a delete block holding
// STRING bytes for a VARBINARY data column, say). Keys of these types therefore match by their bytes.
bool is_equality_delete_byte_type(const DataTypePtr& type) {
const auto primitive_type = remove_nullable(type)->get_primitive_type();
return is_string_type(primitive_type) || is_varbinary(primitive_type);
}

// The rows of a byte-key column. A const column repeats its single value, which can be NULL: when the FE does
// not send Iceberg scan semantics v2, a key column that the data file lacks is filled from
// Const(Nullable(String) NULL), and its rows must match a NULL delete key rather than an empty string.
class ByteKeyColumn {
public:
explicit ByteKeyColumn(const IColumn& column) : _values(&column) {
if (const auto* const_column = check_and_get_column<ColumnConst>(_values)) {
_values = &const_column->get_data_column();
_is_const = true;
}
if (const auto* nullable = check_and_get_column<ColumnNullable>(_values)) {
_null_map = &nullable->get_null_map_data();
_values = &nullable->get_nested_column();
}
}

bool is_null(size_t row) const {
return _null_map != nullptr && (*_null_map)[_is_const ? 0 : row] != 0;
}

StringRef value(size_t row) const { return _values->get_data_at(_is_const ? 0 : row); }

private:
const IColumn* _values;
const NullMap* _null_map = nullptr;
bool _is_const = false;
};

void insert_byte_values(const IColumn& column, HybridSetBase* byte_set) {
const ByteKeyColumn keys(column);
for (size_t row = 0; row < column.size(); ++row) {
if (keys.is_null(row)) {
byte_set->insert(static_cast<const void*>(nullptr));
continue;
}
const StringRef value = keys.value(row);
byte_set->insert(&value);
}
}

void find_byte_values(const IColumn& column, const HybridSetBase& byte_set,
IColumn::Filter* matches) {
const ByteKeyColumn keys(column);
for (size_t row = 0; row < column.size(); ++row) {
if (keys.is_null(row)) {
(*matches)[row] = byte_set.contain_null();
continue;
}
const StringRef value = keys.value(row);
(*matches)[row] = byte_set.find(&value);
}
}

void update_equality_delete_hashes(const ColumnWithTypeAndName& column_and_type,
std::vector<uint64_t>* hashes) {
if (!is_equality_delete_byte_type(column_and_type.type)) {
column_and_type.column->update_hashes_with_value(hashes->data(), nullptr);
return;
}
// Hash the bytes rather than call update_hashes_with_value: VARBINARY does not implement it, and a
// STRING delete key must land on the same hash as the equal VARBINARY data key.
const ByteKeyColumn keys(*column_and_type.column);
for (size_t row = 0; row < column_and_type.column->size(); ++row) {
if (keys.is_null(row)) {
(*hashes)[row] = HashUtil::xxHash64NullWithSeed((*hashes)[row]);
continue;
}
const StringRef value = keys.value(row);
(*hashes)[row] = HashUtil::xxHash64WithSeed(value.data, value.size, (*hashes)[row]);
}
}

bool equality_delete_values_equal(const ColumnWithTypeAndName& data_column, size_t data_row,
const ColumnWithTypeAndName& delete_column, size_t delete_row) {
if (!is_equality_delete_byte_type(data_column.type) ||
!is_equality_delete_byte_type(delete_column.type)) {
return data_column.column->compare_at(data_row, delete_row, *delete_column.column, -1) == 0;
}
const ByteKeyColumn data_keys(*data_column.column);
const ByteKeyColumn delete_keys(*delete_column.column);
if (data_keys.is_null(data_row) || delete_keys.is_null(delete_row)) {
return data_keys.is_null(data_row) && delete_keys.is_null(delete_row);
}
return data_keys.value(data_row) == delete_keys.value(delete_row);
}

} // namespace

std::unique_ptr<EqualityDeleteBase> EqualityDeleteBase::get_delete_impl(
const Block* delete_block, const std::vector<int>& delete_col_ids) {
DCHECK_EQ(delete_block->columns(), delete_col_ids.size());
Expand All @@ -37,15 +137,22 @@ Status SimpleEqualityDelete::_build_set() {
if (_delete_block->columns() != 1) [[unlikely]] {
return Status::InternalError("Simple equality delete can be only applied with one column");
}
auto& column_and_type = _delete_block->get_by_position(0);
const auto& column_and_type = _delete_block->get_by_position(0);
auto delete_column_type = remove_nullable(column_and_type.type)->get_primitive_type();
size_t non_null_rows = _delete_block->rows();
if (const auto* nullable = check_and_get_column<ColumnNullable>(column_and_type.column.get());
nullable != nullptr) {
non_null_rows = std::ranges::count(nullable->get_null_map_data(), UInt8(0));
}
_hybrid_set.reset(create_set(delete_column_type, non_null_rows, true));
_hybrid_set->insert_fixed_len(column_and_type.column, 0);
if (is_equality_delete_byte_type(column_and_type.type)) {
// create_set has no VARBINARY set (it throws NOT_IMPLEMENTED), so every byte key goes into a
// STRING set as bytes, whichever of STRING and VARBINARY the delete file was read as.
_hybrid_set.reset(create_set(TYPE_STRING, non_null_rows, true));
insert_byte_values(*column_and_type.column, _hybrid_set.get());
} else {
_hybrid_set.reset(create_set(delete_column_type, non_null_rows, true));
_hybrid_set->insert_fixed_len(column_and_type.column, 0);
}
return Status::OK();
}

Expand All @@ -61,6 +168,17 @@ Status SimpleEqualityDelete::filter_data_block(
auto block_idx = col_name_to_block_idx->at(block_col_name);

auto column_and_type = data_block->get_by_position(block_idx);
const bool delete_is_byte =
is_equality_delete_byte_type(_delete_block->get_by_position(0).type);
const bool byte_compatible =
delete_is_byte && is_equality_delete_byte_type(column_and_type.type);
// The byte set can only be probed with bytes: an INT data column would be read as StringRef values.
if (delete_is_byte && !byte_compatible) [[unlikely]] {
return Status::InternalError(
"Not support type change in column '{}', src type: {}, target type: {}",
column_and_type.name, _delete_block->get_by_position(0).type->get_name(),
column_and_type.type->get_name());
}

size_t rows = data_block->rows();
// _filter: 1 => in _hybrid_set; 0 => not in _hybrid_set
Expand All @@ -71,7 +189,9 @@ Status SimpleEqualityDelete::filter_data_block(
_single_filter->assign(rows, UInt8(0));
}
auto column = column_and_type.column->convert_to_full_column_if_const();
if (const auto* nullable = check_and_get_column<ColumnNullable>(column.get())) {
if (byte_compatible) {
find_byte_values(*column, *_hybrid_set, _single_filter.get());
} else if (const auto* nullable = check_and_get_column<ColumnNullable>(column.get())) {
const NullMap& null_map = nullable->get_null_map_data();
_hybrid_set->find_batch_nullable(*remove_nullable(column), rows, null_map, *_single_filter);
if (_hybrid_set->contain_null()) {
Expand All @@ -96,8 +216,8 @@ Status MultiEqualityDelete::_build_set() {
size_t rows = _delete_block->rows();
_delete_hashes.clear();
_delete_hashes.resize(rows, 0);
for (ColumnPtr column : _delete_block->get_columns()) {
column->update_hashes_with_value(_delete_hashes.data(), nullptr);
for (const auto& column : _delete_block->get_columns_with_type_and_name()) {
update_equality_delete_hashes(column, &_delete_hashes);
}
for (size_t i = 0; i < rows; ++i) {
_delete_hash_map.insert({_delete_hashes[i], i});
Expand Down Expand Up @@ -126,7 +246,11 @@ Status MultiEqualityDelete::filter_data_block(
}
auto column_and_type =
data_block->safe_get_by_position(col_name_to_block_idx->at(block_column_name));
if (!delete_col.type->equals(*column_and_type.type)) [[unlikely]] {
// A STRING delete key may meet a VARBINARY data column for the same Iceberg BINARY field: both are
// hashed and compared as bytes, so only a change to or from a non-byte type is rejected.
const bool byte_compatible = is_equality_delete_byte_type(delete_col.type) &&
is_equality_delete_byte_type(column_and_type.type);
if (!delete_col.type->equals(*column_and_type.type) && !byte_compatible) [[unlikely]] {
return Status::InternalError(
"Not support type change in column '{}', src type: {}, target type: {}",
block_column_name, delete_col.type->get_name(),
Expand All @@ -138,8 +262,7 @@ Status MultiEqualityDelete::filter_data_block(
_data_hashes.clear();
_data_hashes.resize(rows, 0);
for (size_t index : _data_column_index) {
data_block->get_by_position(index).column->update_hashes_with_value(_data_hashes.data(),
nullptr);
update_equality_delete_hashes(data_block->get_by_position(index), &_data_hashes);
}
auto* filter_data = filter.data();
for (size_t i = 0; i < rows; ++i) {
Expand All @@ -159,9 +282,9 @@ Status MultiEqualityDelete::filter_data_block(
bool MultiEqualityDelete::_equal(Block* data_block, size_t data_row_index,
size_t delete_row_index) {
for (size_t i = 0; i < _delete_block->columns(); ++i) {
ColumnPtr data_col = data_block->get_by_position(_data_column_index[i]).column;
ColumnPtr delete_col = _delete_block->get_by_position(i).column;
if (data_col->compare_at(data_row_index, delete_row_index, *delete_col, -1) != 0) {
if (!equality_delete_values_equal(data_block->get_by_position(_data_column_index[i]),
data_row_index, _delete_block->get_by_position(i),
delete_row_index)) {
return false;
}
}
Expand Down
Loading
Loading