Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,13 @@ void set_iceberg_delete_field_id(ColumnDefinition* column) {
}

class PositionDeleteFileTableReader final : public format::TableReader {
public:
explicit PositionDeleteFileTableReader(const TFileEncryptionInfo* encryption)
: _encryption(encryption) {}

protected:
const TFileEncryptionInfo* file_encryption_info() const override { return _encryption; }

format::TableColumnMappingMode mapping_mode() const override {
const bool has_field_ids = supports_iceberg_scan_semantics_v1(_scan_params)
? schema_has_any_field_id(_data_reader.file_schema)
Expand All @@ -157,6 +163,9 @@ class PositionDeleteFileTableReader final : public format::TableReader {
supports_iceberg_scan_semantics_v1(_scan_params) &&
_format == format::FileFormat::PARQUET;
}

private:
const TFileEncryptionInfo* _encryption;
};

} // namespace
Expand Down Expand Up @@ -277,6 +286,12 @@ Status IcebergPositionDeleteSysTableV2Reader::_init_split() {
"file");
}
_delete_file_desc = _iceberg_file_desc->delete_files.data();
if (_delete_file_desc->__isset.encryption_info &&
(is_iceberg_deletion_vector(*_delete_file_desc) ||
!_delete_file_desc->__isset.file_format ||
_delete_file_desc->file_format != TFileFormatType::FORMAT_PARQUET)) {
return Status::NotSupported("File encryption [UNSUPPORTED_FILE_FORMAT]");
}
if (is_iceberg_deletion_vector(*_delete_file_desc)) {
_delete_file_kind = DeleteFileKind::DELETION_VECTOR;
} else if (_delete_file_desc->__isset.content &&
Expand Down Expand Up @@ -334,7 +349,10 @@ Status IcebergPositionDeleteSysTableV2Reader::_init_position_delete_reader() {
_position_profile_cache = _standalone_position_profile_cache.get();
}
}
_position_reader = std::make_unique<PositionDeleteFileTableReader>();
// The metadata table scans the delete file itself; a data-file descriptor is unrelated.
_position_reader = std::make_unique<PositionDeleteFileTableReader>(
_delete_file_desc->__isset.encryption_info ? &_delete_file_desc->encryption_info
: nullptr);
RETURN_IF_ERROR(_position_reader->init({
.projected_columns = std::move(projected_columns),
.conjuncts = {},
Expand Down
93 changes: 73 additions & 20 deletions be/src/format_v2/table/iceberg_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1571,6 +1571,23 @@ Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options
}
}
}
bool encrypted = file_encryption_info() != nullptr;
if (_iceberg_params.has_value() && _iceberg_params->__isset.delete_files) {
for (const auto& delete_file : _iceberg_params->delete_files) {
if (!delete_file.__isset.encryption_info) {
continue;
}
encrypted = true;
if (is_iceberg_deletion_vector(delete_file) || !delete_file.__isset.file_format ||
delete_file.file_format != TFileFormatType::FORMAT_PARQUET) {
return Status::NotSupported("File encryption [UNSUPPORTED_FILE_FORMAT]");
}
}
}
if (encrypted && _push_down_agg_type != TPushAggOp::type::NONE) {
// Metadata COUNT must not bypass a delete file's independent key or authentication.
return Status::NotSupported("File encryption [AGGREGATE_UNAVAILABLE]");
}
RETURN_IF_ERROR(TableReader::prepare_split(options));
SCOPED_TIMER(_profile.total_timer);
SCOPED_TIMER(_profile.prepare_split_timer);
Expand Down Expand Up @@ -1734,6 +1751,10 @@ Status IcebergTableReader::_parse_deletion_vector_file(const TTableFormatFileDes
if (deletion_vector == nullptr) {
return Status::OK();
}
if (deletion_vector->__isset.encryption_info) {
// PME applies to Parquet modules, not Puffin deletion-vector blobs.
return Status::NotSupported("File encryption [UNSUPPORTED_FILE_FORMAT]");
}
size_t bytes_read = 0;
RETURN_IF_ERROR(validate_iceberg_deletion_vector_descriptor(*deletion_vector, bytes_read));

Expand Down Expand Up @@ -2168,6 +2189,14 @@ Status IcebergTableReader::_convert_equality_delete_filter(const EqualityDeleteF
DORIS_CHECK(_split_cache != nullptr);
DORIS_CHECK(result != nullptr);
DORIS_CHECK(comparison_types.size() == filter.delete_key_types.size());
if (filter.cache_key.empty()) {
// Converted equality keys are plaintext too. An uncached encrypted filter cannot share
// the empty identity with another file or bypass a later split's key validation.
RETURN_IF_ERROR(_build_converted_equality_delete_filter(filter, comparison_types, result));
COUNTER_UPDATE(_profile.equality_delete_hash_index_memory,
static_cast<int64_t>(result->hash_index->memory_usage()));
return Status::OK();
}
std::ostringstream cache_key;
cache_key << filter.cache_key << ":compare";
for (const auto& type : comparison_types) {
Expand Down Expand Up @@ -2234,6 +2263,10 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe
if (!delete_file.__isset.file_format) {
return Status::InternalError("Iceberg delete file is missing file format");
}
if (delete_file.__isset.encryption_info &&
delete_file.file_format != TFileFormatType::FORMAT_PARQUET) {
return Status::NotSupported("File encryption [UNSUPPORTED_FILE_FORMAT]");
}
if (delete_file.file_format != TFileFormatType::FORMAT_PARQUET &&
delete_file.file_format != TFileFormatType::FORMAT_ORC) {
return Status::NotSupported("Unsupported Iceberg delete file format {}",
Expand All @@ -2257,7 +2290,9 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe
*reader = std::make_unique<format::parquet::ParquetReader>(
system_properties, file_description, io_ctx, _scanner_profile, std::nullopt,
enable_mapping_timestamp_tz, enable_mapping_varbinary, "", preserve_binary_uuid(),
_parquet_profile_cache);
_parquet_profile_cache,
delete_file.__isset.encryption_info ? std::optional(delete_file.encryption_info)
: std::nullopt);
} else {
*reader = std::make_unique<format::orc::OrcReader>(
system_properties, file_description, io_ctx, _scanner_profile, std::nullopt,
Expand Down Expand Up @@ -2331,25 +2366,32 @@ Status IcebergTableReader::_init_position_delete_rows(
IcebergDeleteFileIOContext delete_io_ctx(_runtime_state);
for (const auto& delete_file : delete_files) {
Status read_status = Status::OK();
// A position delete file normally references many data files. Cache the complete
// path-to-position map once; caching only the current data file would still rescan the
// shared delete file for every subsequent split.
auto rows_by_data_file = _split_cache->get<PositionDeleteRowsCollector::PositionDeleteFile>(
_delete_file_cache_key("iceberg_v2_position_delete_", delete_file.path),
[&]() -> std::shared_ptr<PositionDeleteRowsCollector::PositionDeleteFile> {
auto result =
std::make_shared<PositionDeleteRowsCollector::PositionDeleteFile>();
PositionDeleteRowsCollector collector(result.get());
read_status = _read_position_delete_file(delete_file, delete_scan_params,
&delete_io_ctx, &collector);
if (!read_status.ok()) {
return nullptr;
}
for (auto& [_, rows] : *result) {
std::ranges::sort(rows);
}
return result;
});
auto load_rows = [&]() -> std::shared_ptr<PositionDeleteRowsCollector::PositionDeleteFile> {
auto result = std::make_shared<PositionDeleteRowsCollector::PositionDeleteFile>();
PositionDeleteRowsCollector collector(result.get());
read_status = _read_position_delete_file(delete_file, delete_scan_params,
&delete_io_ctx, &collector);
if (!read_status.ok()) {
return nullptr;
}
for (auto& [_, rows] : *result) {
std::ranges::sort(rows);
}
return result;
};
// Keep the cache value alive while another scanner may release the cache's ownership.
std::shared_ptr<PositionDeleteRowsCollector::PositionDeleteFile> rows_by_data_file;
if (delete_file.__isset.encryption_info) {
// A warmed path must never authorize another split's missing, wrong or expired key.
// Keep decrypted positions scan-local until encrypted cache identities are defined.
rows_by_data_file = load_rows();
} else {
// Plaintext position files commonly reference many data files; retain their complete
// path-to-position cache so subsequent splits do not rescan the shared delete file.
rows_by_data_file = _split_cache->get<PositionDeleteRowsCollector::PositionDeleteFile>(
_delete_file_cache_key("iceberg_v2_position_delete_", delete_file.path),
load_rows);
}
RETURN_IF_ERROR(read_status);
DORIS_CHECK(rows_by_data_file != nullptr);
const auto rows_it = rows_by_data_file->find(_data_file_path());
Expand Down Expand Up @@ -2528,6 +2570,17 @@ Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDe
if (!delete_file.__isset.field_ids || delete_file.field_ids.empty()) {
return Status::InternalError("Iceberg equality delete file is missing field ids");
}
if (delete_file.__isset.encryption_info) {
EqualityDeleteFilter filter;
RETURN_IF_ERROR(
_load_equality_delete_file(delete_file, scan_params, delete_io_ctx, &filter));
// Leaving cache_key empty also prevents type-converted equality keys from being shared.
COUNTER_UPDATE(_profile.equality_delete_index_cache_miss_count, 1);
COUNTER_UPDATE(_profile.equality_delete_hash_index_memory,
static_cast<int64_t>(filter.hash_index->memory_usage()));
_equality_delete_filters.push_back(std::move(filter));
return Status::OK();
}
std::ostringstream cache_key;
cache_key << _delete_file_cache_key("iceberg_v2_equality_delete_", delete_file.path);
if (scan_params.__isset.current_schema_id) {
Expand Down
9 changes: 8 additions & 1 deletion be/src/format_v2/table/iceberg_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ class IcebergTableReader : public format::TableReader {
// Iceberg UUID uses the same 16-byte STRING/VARBINARY carrier in data, defaults and deletes.
bool preserve_binary_uuid() const override { return true; }

const TFileEncryptionInfo* file_encryption_info() const override {
return _iceberg_params.has_value() && _iceberg_params->__isset.encryption_info
? &_iceberg_params->encryption_info
: nullptr;
}

Status validate_file_mapping(const format::TableColumnMapper& mapper) const override;

// Iceberg's Parquet reader resolves a data file that carries no field ids at all through the
Expand Down Expand Up @@ -266,7 +272,8 @@ class IcebergTableReader : public format::TableReader {
std::vector<DataTypePtr> delete_key_types;
Block delete_block;
std::shared_ptr<const EqualityDeleteHashIndex> hash_index;
// Split-cache key of this filter, extended by the comparison types for converted copies.
// Empty for encrypted filters: neither the original nor converted plaintext may be cached.
// Other filters extend this split-cache key with the comparison types for converted copies.
std::string cache_key;
};
std::vector<EqualityDeleteFilter> _equality_delete_filters;
Expand Down
7 changes: 6 additions & 1 deletion be/src/format_v2/table_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1318,6 +1318,10 @@ Status TableReader::create_next_reader(bool* eos) {

Status TableReader::create_file_reader(std::unique_ptr<FileReader>* reader) {
DORIS_CHECK(reader != nullptr);
const auto* encryption = file_encryption_info();
if (encryption != nullptr && _format != FileFormat::PARQUET) {
return Status::NotSupported("File encryption [UNSUPPORTED_FILE_FORMAT]");
}
const bool enable_mapping_timestamp_tz = _scan_params != nullptr &&
_scan_params->__isset.enable_mapping_timestamp_tz &&
_scan_params->enable_mapping_timestamp_tz;
Expand All @@ -1333,7 +1337,8 @@ Status TableReader::create_file_reader(std::unique_ptr<FileReader>* reader) {
*reader = std::make_unique<format::parquet::ParquetReader>(
_system_properties, _current_task->data_file, _io_ctx, _scanner_profile,
_global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary,
hive_parquet_time_zone, preserve_binary_uuid(), _parquet_profile_cache);
hive_parquet_time_zone, preserve_binary_uuid(), _parquet_profile_cache,
encryption == nullptr ? std::nullopt : std::optional(*encryption));
return Status::OK();
}
if (_format == FileFormat::ORC) {
Expand Down
3 changes: 3 additions & 0 deletions be/src/format_v2/table_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,9 @@ class TableReader {
virtual Status create_file_reader(std::unique_ptr<FileReader>* reader);
// Some table formats expose UUID as raw bytes rather than the native Doris UUID type.
virtual bool preserve_binary_uuid() const { return false; }

// The descriptor belongs to the current split. File readers copy it before initialization.
virtual const TFileEncryptionInfo* file_encryption_info() const { return nullptr; }
virtual TableColumnMappingMode mapping_mode() const { return TableColumnMappingMode::BY_NAME; }
virtual void configure_mapper_options(TableColumnMapperOptions*) const {}
virtual Status annotate_file_schema(std::vector<ColumnDefinition>* file_schema) {
Expand Down
Loading
Loading