From 5b5a0b3b2f89d6acb6203e725c45d088dc1e60cb Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 23 Oct 2026 11:51:06 +0800 Subject: [PATCH] [fix](be) Match Iceberg BINARY and FIXED equality delete keys by bytes ### What problem does this PR solve? Issue Number: None Related PR: #66538, #68786 Problem Summary: Since #68786 the BE reads Iceberg BINARY and FIXED columns as VARBINARY. The file scanner V1 (enable_file_scanner_v2 = false) cannot apply an equality delete whose key is such a column. With one key, create_set(TYPE_VARBINARY) throws NOT_IMPLEMENTED. With several keys, ColumnVarbinary does not implement update_hashes_with_value, so the read fails too. And a delete key read as STRING for a VARBINARY data column, or the other way round, is rejected as a type change. branch-4.1 and branch-4.2 (#66538) match string and VARBINARY keys by their bytes. This change does the same: a single byte key goes into a STRING set as bytes, composite byte keys are hashed with xxhash over their bytes and compared as bytes, and STRING and VARBINARY keys count as the same type. Unlike branch-4.2, the byte path also reads const key columns: when the FE does not send Iceberg scan semantics v2, a key column that the data file lacks is filled from Const(NULL), and those rows must match a NULL delete key, not an empty string. ### Release note Fix: With enable_file_scanner_v2 = false, reading an Iceberg table whose equality deletes use a BINARY or FIXED key no longer fails. ### Check List (For Author) - Test: Unit Test - be/test/format/table/equality_delete_test.cpp: the 5 tests of branch-4.2 (single and composite VARBINARY keys, STRING delete keys for VARBINARY data and the reverse, NULL keys) and a composite key whose data column is a const NULL. All 6 fail with the current equality_delete.cpp (one aborts on a bad cast) and pass with this change - IcebergReaderTest, ParquetReadLinesTest, IcebergPositionDeleteSysTable*Test and VIcebergDeleteSinkTest pass - Behavior changed: No - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) --- be/src/format/table/equality_delete.cpp | 147 +++++++++++-- be/test/format/table/equality_delete_test.cpp | 193 ++++++++++++++++++ 2 files changed, 328 insertions(+), 12 deletions(-) create mode 100644 be/test/format/table/equality_delete_test.cpp diff --git a/be/src/format/table/equality_delete.cpp b/be/src/format/table/equality_delete.cpp index 2386e8c1bc106a..21071a89114957 100644 --- a/be/src/format/table/equality_delete.cpp +++ b/be/src/format/table/equality_delete.cpp @@ -17,11 +17,111 @@ #include "format/table/equality_delete.h" +#include "core/column/column_const.h" #include "core/column/column_nullable.h" #include "exprs/create_predicate_function.h" +#include "util/hash_util.hpp" namespace doris { +namespace { + +// Iceberg BINARY / FIXED keys are read as VARBINARY and string keys as STRING, and the delete file and +// the data file may carry the same key in different ones of these classes (a delete block holding +// STRING bytes for a VARBINARY data column, say). Keys of these types therefore match by their bytes. +bool is_equality_delete_byte_type(const DataTypePtr& type) { + const auto primitive_type = remove_nullable(type)->get_primitive_type(); + return is_string_type(primitive_type) || is_varbinary(primitive_type); +} + +// The rows of a byte-key column. A const column repeats its single value, which can be NULL: when the FE does +// not send Iceberg scan semantics v2, a key column that the data file lacks is filled from +// Const(Nullable(String) NULL), and its rows must match a NULL delete key rather than an empty string. +class ByteKeyColumn { +public: + explicit ByteKeyColumn(const IColumn& column) : _values(&column) { + if (const auto* const_column = check_and_get_column(_values)) { + _values = &const_column->get_data_column(); + _is_const = true; + } + if (const auto* nullable = check_and_get_column(_values)) { + _null_map = &nullable->get_null_map_data(); + _values = &nullable->get_nested_column(); + } + } + + bool is_null(size_t row) const { + return _null_map != nullptr && (*_null_map)[_is_const ? 0 : row] != 0; + } + + StringRef value(size_t row) const { return _values->get_data_at(_is_const ? 0 : row); } + +private: + const IColumn* _values; + const NullMap* _null_map = nullptr; + bool _is_const = false; +}; + +void insert_byte_values(const IColumn& column, HybridSetBase* byte_set) { + const ByteKeyColumn keys(column); + for (size_t row = 0; row < column.size(); ++row) { + if (keys.is_null(row)) { + byte_set->insert(static_cast(nullptr)); + continue; + } + const StringRef value = keys.value(row); + byte_set->insert(&value); + } +} + +void find_byte_values(const IColumn& column, const HybridSetBase& byte_set, + IColumn::Filter* matches) { + const ByteKeyColumn keys(column); + for (size_t row = 0; row < column.size(); ++row) { + if (keys.is_null(row)) { + (*matches)[row] = byte_set.contain_null(); + continue; + } + const StringRef value = keys.value(row); + (*matches)[row] = byte_set.find(&value); + } +} + +void update_equality_delete_hashes(const ColumnWithTypeAndName& column_and_type, + std::vector* hashes) { + if (!is_equality_delete_byte_type(column_and_type.type)) { + column_and_type.column->update_hashes_with_value(hashes->data(), nullptr); + return; + } + // Hash the bytes rather than call update_hashes_with_value: VARBINARY does not implement it, and a + // STRING delete key must land on the same hash as the equal VARBINARY data key. + const ByteKeyColumn keys(*column_and_type.column); + for (size_t row = 0; row < column_and_type.column->size(); ++row) { + if (keys.is_null(row)) { + (*hashes)[row] = HashUtil::xxHash64NullWithSeed((*hashes)[row]); + continue; + } + const StringRef value = keys.value(row); + (*hashes)[row] = HashUtil::xxHash64WithSeed(value.data, value.size, (*hashes)[row]); + } +} + +bool equality_delete_values_equal(const ColumnWithTypeAndName& data_column, size_t data_row, + const ColumnWithTypeAndName& delete_column, size_t delete_row) { + if (!is_equality_delete_byte_type(data_column.type) || + !is_equality_delete_byte_type(delete_column.type)) { + return data_column.column->compare_at(data_row, delete_row, *delete_column.column, -1) == 0; + } + const ByteKeyColumn data_keys(*data_column.column); + const ByteKeyColumn delete_keys(*delete_column.column); + if (data_keys.is_null(data_row) || delete_keys.is_null(delete_row)) { + return data_keys.is_null(data_row) && delete_keys.is_null(delete_row); + } + return data_keys.value(data_row) == delete_keys.value(delete_row); +} + +} // namespace + std::unique_ptr EqualityDeleteBase::get_delete_impl( const Block* delete_block, const std::vector& delete_col_ids) { DCHECK_EQ(delete_block->columns(), delete_col_ids.size()); @@ -37,15 +137,22 @@ Status SimpleEqualityDelete::_build_set() { if (_delete_block->columns() != 1) [[unlikely]] { return Status::InternalError("Simple equality delete can be only applied with one column"); } - auto& column_and_type = _delete_block->get_by_position(0); + const auto& column_and_type = _delete_block->get_by_position(0); auto delete_column_type = remove_nullable(column_and_type.type)->get_primitive_type(); size_t non_null_rows = _delete_block->rows(); if (const auto* nullable = check_and_get_column(column_and_type.column.get()); nullable != nullptr) { non_null_rows = std::ranges::count(nullable->get_null_map_data(), UInt8(0)); } - _hybrid_set.reset(create_set(delete_column_type, non_null_rows, true)); - _hybrid_set->insert_fixed_len(column_and_type.column, 0); + if (is_equality_delete_byte_type(column_and_type.type)) { + // create_set has no VARBINARY set (it throws NOT_IMPLEMENTED), so every byte key goes into a + // STRING set as bytes, whichever of STRING and VARBINARY the delete file was read as. + _hybrid_set.reset(create_set(TYPE_STRING, non_null_rows, true)); + insert_byte_values(*column_and_type.column, _hybrid_set.get()); + } else { + _hybrid_set.reset(create_set(delete_column_type, non_null_rows, true)); + _hybrid_set->insert_fixed_len(column_and_type.column, 0); + } return Status::OK(); } @@ -61,6 +168,17 @@ Status SimpleEqualityDelete::filter_data_block( auto block_idx = col_name_to_block_idx->at(block_col_name); auto column_and_type = data_block->get_by_position(block_idx); + const bool delete_is_byte = + is_equality_delete_byte_type(_delete_block->get_by_position(0).type); + const bool byte_compatible = + delete_is_byte && is_equality_delete_byte_type(column_and_type.type); + // The byte set can only be probed with bytes: an INT data column would be read as StringRef values. + if (delete_is_byte && !byte_compatible) [[unlikely]] { + return Status::InternalError( + "Not support type change in column '{}', src type: {}, target type: {}", + column_and_type.name, _delete_block->get_by_position(0).type->get_name(), + column_and_type.type->get_name()); + } size_t rows = data_block->rows(); // _filter: 1 => in _hybrid_set; 0 => not in _hybrid_set @@ -71,7 +189,9 @@ Status SimpleEqualityDelete::filter_data_block( _single_filter->assign(rows, UInt8(0)); } auto column = column_and_type.column->convert_to_full_column_if_const(); - if (const auto* nullable = check_and_get_column(column.get())) { + if (byte_compatible) { + find_byte_values(*column, *_hybrid_set, _single_filter.get()); + } else if (const auto* nullable = check_and_get_column(column.get())) { const NullMap& null_map = nullable->get_null_map_data(); _hybrid_set->find_batch_nullable(*remove_nullable(column), rows, null_map, *_single_filter); if (_hybrid_set->contain_null()) { @@ -96,8 +216,8 @@ Status MultiEqualityDelete::_build_set() { size_t rows = _delete_block->rows(); _delete_hashes.clear(); _delete_hashes.resize(rows, 0); - for (ColumnPtr column : _delete_block->get_columns()) { - column->update_hashes_with_value(_delete_hashes.data(), nullptr); + for (const auto& column : _delete_block->get_columns_with_type_and_name()) { + update_equality_delete_hashes(column, &_delete_hashes); } for (size_t i = 0; i < rows; ++i) { _delete_hash_map.insert({_delete_hashes[i], i}); @@ -126,7 +246,11 @@ Status MultiEqualityDelete::filter_data_block( } auto column_and_type = data_block->safe_get_by_position(col_name_to_block_idx->at(block_column_name)); - if (!delete_col.type->equals(*column_and_type.type)) [[unlikely]] { + // A STRING delete key may meet a VARBINARY data column for the same Iceberg BINARY field: both are + // hashed and compared as bytes, so only a change to or from a non-byte type is rejected. + const bool byte_compatible = is_equality_delete_byte_type(delete_col.type) && + is_equality_delete_byte_type(column_and_type.type); + if (!delete_col.type->equals(*column_and_type.type) && !byte_compatible) [[unlikely]] { return Status::InternalError( "Not support type change in column '{}', src type: {}, target type: {}", block_column_name, delete_col.type->get_name(), @@ -138,8 +262,7 @@ Status MultiEqualityDelete::filter_data_block( _data_hashes.clear(); _data_hashes.resize(rows, 0); for (size_t index : _data_column_index) { - data_block->get_by_position(index).column->update_hashes_with_value(_data_hashes.data(), - nullptr); + update_equality_delete_hashes(data_block->get_by_position(index), &_data_hashes); } auto* filter_data = filter.data(); for (size_t i = 0; i < rows; ++i) { @@ -159,9 +282,9 @@ Status MultiEqualityDelete::filter_data_block( bool MultiEqualityDelete::_equal(Block* data_block, size_t data_row_index, size_t delete_row_index) { for (size_t i = 0; i < _delete_block->columns(); ++i) { - ColumnPtr data_col = data_block->get_by_position(_data_column_index[i]).column; - ColumnPtr delete_col = _delete_block->get_by_position(i).column; - if (data_col->compare_at(data_row_index, delete_row_index, *delete_col, -1) != 0) { + if (!equality_delete_values_equal(data_block->get_by_position(_data_column_index[i]), + data_row_index, _delete_block->get_by_position(i), + delete_row_index)) { return false; } } diff --git a/be/test/format/table/equality_delete_test.cpp b/be/test/format/table/equality_delete_test.cpp new file mode 100644 index 00000000000000..8e4233cb98ea08 --- /dev/null +++ b/be/test/format/table/equality_delete_test.cpp @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/table/equality_delete.h" + +#include + +#include +#include +#include +#include + +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_varbinary.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_varbinary.h" + +namespace doris { + +namespace { + +ColumnWithTypeAndName nullable_varbinary_column( + const std::string& name, const std::vector>& values) { + auto data = ColumnVarbinary::create(); + auto null_map = ColumnUInt8::create(); + for (const auto& value : values) { + const std::string bytes = value.value_or(""); + data->insert_data(bytes.data(), bytes.size()); + null_map->insert_value(!value.has_value()); + } + return {ColumnNullable::create(std::move(data), std::move(null_map)), + make_nullable(std::make_shared()), name}; +} + +ColumnWithTypeAndName varbinary_column(const std::string& name, + const std::vector& values) { + auto data = ColumnVarbinary::create(); + for (const auto& value : values) { + data->insert_data(value.data(), value.size()); + } + return {std::move(data), std::make_shared(), name}; +} + +ColumnWithTypeAndName nullable_string_column( + const std::string& name, const std::vector>& values) { + auto data = ColumnString::create(); + auto null_map = ColumnUInt8::create(); + for (const auto& value : values) { + const std::string bytes = value.value_or(""); + data->insert_data(bytes.data(), bytes.size()); + null_map->insert_value(!value.has_value()); + } + return {ColumnNullable::create(std::move(data), std::move(null_map)), + make_nullable(std::make_shared()), name}; +} + +ColumnWithTypeAndName string_column(const std::string& name, + const std::vector& values) { + auto data = ColumnString::create(); + for (const auto& value : values) { + data->insert_data(value.data(), value.size()); + } + return {std::move(data), std::make_shared(), name}; +} + +ColumnWithTypeAndName int_column(const std::string& name, const std::vector& values) { + auto data = ColumnInt32::create(); + data->get_data().assign(values.begin(), values.end()); + return {std::move(data), std::make_shared(), name}; +} + +std::vector apply_equality_delete( + const Block& delete_block, const std::vector& field_ids, Block* data_block, + const std::unordered_map& column_indexes, + const std::unordered_map& field_names) { + RuntimeProfile profile("equality_delete_varbinary"); + auto equality_delete = EqualityDeleteBase::get_delete_impl(&delete_block, field_ids); + EXPECT_TRUE(equality_delete->init(&profile).ok()); + IColumn::Filter filter(data_block->rows(), 1); + EXPECT_TRUE(equality_delete->filter_data_block(data_block, &column_indexes, field_names, filter) + .ok()); + return {filter.begin(), filter.end()}; +} + +} // namespace + +TEST(EqualityDeleteTest, NullableSingleVarbinaryKeyUsesByteHashing) { + Block delete_block; + delete_block.insert(nullable_varbinary_column( + "binary_key", {std::nullopt, std::string("delete\0value", 12)})); + Block data_block; + data_block.insert(nullable_varbinary_column( + "binary_key", + {std::nullopt, std::string("keep\0value", 10), std::string("delete\0value", 12)})); + + EXPECT_EQ((std::vector {0, 1, 0}), + apply_equality_delete(delete_block, {7}, &data_block, {{"binary_key", 0}}, + {{7, "binary_key"}})); +} + +TEST(EqualityDeleteTest, NullableStringDeleteMatchesVarbinaryData) { + Block delete_block; + delete_block.insert( + nullable_string_column("binary_key", {std::nullopt, std::string("delete\0value", 12)})); + Block data_block; + data_block.insert(nullable_varbinary_column( + "binary_key", + {std::nullopt, std::string("keep\0value", 10), std::string("delete\0value", 12)})); + + EXPECT_EQ((std::vector {0, 1, 0}), + apply_equality_delete(delete_block, {7}, &data_block, {{"binary_key", 0}}, + {{7, "binary_key"}})); +} + +TEST(EqualityDeleteTest, NullableVarbinaryDeleteMatchesStringData) { + Block delete_block; + delete_block.insert(nullable_varbinary_column( + "binary_key", {std::nullopt, std::string("delete\0value", 12)})); + Block data_block; + data_block.insert(nullable_string_column( + "binary_key", + {std::nullopt, std::string("keep\0value", 10), std::string("delete\0value", 12)})); + + EXPECT_EQ((std::vector {0, 1, 0}), + apply_equality_delete(delete_block, {7}, &data_block, {{"binary_key", 0}}, + {{7, "binary_key"}})); +} + +TEST(EqualityDeleteTest, CompositeVarbinaryKeyUsesByteHashing) { + Block delete_block; + delete_block.insert(varbinary_column("binary_key", {"same", "other"})); + delete_block.insert(int_column("version", {1, 2})); + Block data_block; + data_block.insert(varbinary_column("binary_key", {"same", "other", "same"})); + data_block.insert(int_column("version", {2, 2, 3})); + + EXPECT_EQ((std::vector {1, 0, 1}), + apply_equality_delete(delete_block, {7, 8}, &data_block, + {{"binary_key", 0}, {"version", 1}}, + {{7, "binary_key"}, {8, "version"}})); +} + +TEST(EqualityDeleteTest, CompositeStringDeleteMatchesVarbinaryData) { + Block delete_block; + delete_block.insert(string_column("binary_key", {std::string("same\0bytes", 10), "other"})); + delete_block.insert(int_column("version", {1, 2})); + Block data_block; + data_block.insert( + varbinary_column("binary_key", {std::string("same\0bytes", 10), "other", "other"})); + data_block.insert(int_column("version", {2, 2, 3})); + + EXPECT_EQ((std::vector {1, 0, 1}), + apply_equality_delete(delete_block, {7, 8}, &data_block, + {{"binary_key", 0}, {"version", 1}}, + {{7, "binary_key"}, {8, "version"}})); +} + +TEST(EqualityDeleteTest, CompositeConstNullStringKeyMatchesNullDeleteKey) { + Block delete_block; + delete_block.insert(nullable_string_column("string_key", {std::nullopt, std::string()})); + delete_block.insert(int_column("version", {2, 3})); + // A key column that the data file lacks can arrive as one const NULL for all rows. + auto const_null = nullable_string_column("string_key", {std::nullopt}); + Block data_block; + data_block.insert({ColumnConst::create(const_null.column, 3), const_null.type, "string_key"}); + data_block.insert(int_column("version", {1, 2, 3})); + + EXPECT_EQ((std::vector {1, 0, 1}), + apply_equality_delete(delete_block, {7, 8}, &data_block, + {{"string_key", 0}, {"version", 1}}, + {{7, "string_key"}, {8, "version"}})); +} + +} // namespace doris