diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index c84cf23743fc36..1e8a568934baf3 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -136,7 +136,13 @@ void set_iceberg_delete_field_id(ColumnDefinition* column) { } class PositionDeleteFileTableReader final : public format::TableReader { +public: + explicit PositionDeleteFileTableReader(const TFileEncryptionInfo* encryption) + : _encryption(encryption) {} + protected: + const TFileEncryptionInfo* file_encryption_info() const override { return _encryption; } + format::TableColumnMappingMode mapping_mode() const override { const bool has_field_ids = supports_iceberg_scan_semantics_v1(_scan_params) ? schema_has_any_field_id(_data_reader.file_schema) @@ -157,6 +163,9 @@ class PositionDeleteFileTableReader final : public format::TableReader { supports_iceberg_scan_semantics_v1(_scan_params) && _format == format::FileFormat::PARQUET; } + +private: + const TFileEncryptionInfo* _encryption; }; } // namespace @@ -277,6 +286,12 @@ Status IcebergPositionDeleteSysTableV2Reader::_init_split() { "file"); } _delete_file_desc = _iceberg_file_desc->delete_files.data(); + if (_delete_file_desc->__isset.encryption_info && + (is_iceberg_deletion_vector(*_delete_file_desc) || + !_delete_file_desc->__isset.file_format || + _delete_file_desc->file_format != TFileFormatType::FORMAT_PARQUET)) { + return Status::NotSupported("File encryption [UNSUPPORTED_FILE_FORMAT]"); + } if (is_iceberg_deletion_vector(*_delete_file_desc)) { _delete_file_kind = DeleteFileKind::DELETION_VECTOR; } else if (_delete_file_desc->__isset.content && @@ -334,7 +349,10 @@ Status IcebergPositionDeleteSysTableV2Reader::_init_position_delete_reader() { _position_profile_cache = _standalone_position_profile_cache.get(); } } - _position_reader = std::make_unique(); + // The metadata table scans the delete file itself; a data-file descriptor is unrelated. + _position_reader = std::make_unique( + _delete_file_desc->__isset.encryption_info ? &_delete_file_desc->encryption_info + : nullptr); RETURN_IF_ERROR(_position_reader->init({ .projected_columns = std::move(projected_columns), .conjuncts = {}, diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 819d2dc03edcf5..8d9412c2392a12 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -1571,6 +1571,23 @@ Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options } } } + bool encrypted = file_encryption_info() != nullptr; + if (_iceberg_params.has_value() && _iceberg_params->__isset.delete_files) { + for (const auto& delete_file : _iceberg_params->delete_files) { + if (!delete_file.__isset.encryption_info) { + continue; + } + encrypted = true; + if (is_iceberg_deletion_vector(delete_file) || !delete_file.__isset.file_format || + delete_file.file_format != TFileFormatType::FORMAT_PARQUET) { + return Status::NotSupported("File encryption [UNSUPPORTED_FILE_FORMAT]"); + } + } + } + if (encrypted && _push_down_agg_type != TPushAggOp::type::NONE) { + // Metadata COUNT must not bypass a delete file's independent key or authentication. + return Status::NotSupported("File encryption [AGGREGATE_UNAVAILABLE]"); + } RETURN_IF_ERROR(TableReader::prepare_split(options)); SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.prepare_split_timer); @@ -1734,6 +1751,10 @@ Status IcebergTableReader::_parse_deletion_vector_file(const TTableFormatFileDes if (deletion_vector == nullptr) { return Status::OK(); } + if (deletion_vector->__isset.encryption_info) { + // PME applies to Parquet modules, not Puffin deletion-vector blobs. + return Status::NotSupported("File encryption [UNSUPPORTED_FILE_FORMAT]"); + } size_t bytes_read = 0; RETURN_IF_ERROR(validate_iceberg_deletion_vector_descriptor(*deletion_vector, bytes_read)); @@ -2168,6 +2189,14 @@ Status IcebergTableReader::_convert_equality_delete_filter(const EqualityDeleteF DORIS_CHECK(_split_cache != nullptr); DORIS_CHECK(result != nullptr); DORIS_CHECK(comparison_types.size() == filter.delete_key_types.size()); + if (filter.cache_key.empty()) { + // Converted equality keys are plaintext too. An uncached encrypted filter cannot share + // the empty identity with another file or bypass a later split's key validation. + RETURN_IF_ERROR(_build_converted_equality_delete_filter(filter, comparison_types, result)); + COUNTER_UPDATE(_profile.equality_delete_hash_index_memory, + static_cast(result->hash_index->memory_usage())); + return Status::OK(); + } std::ostringstream cache_key; cache_key << filter.cache_key << ":compare"; for (const auto& type : comparison_types) { @@ -2234,6 +2263,10 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe if (!delete_file.__isset.file_format) { return Status::InternalError("Iceberg delete file is missing file format"); } + if (delete_file.__isset.encryption_info && + delete_file.file_format != TFileFormatType::FORMAT_PARQUET) { + return Status::NotSupported("File encryption [UNSUPPORTED_FILE_FORMAT]"); + } if (delete_file.file_format != TFileFormatType::FORMAT_PARQUET && delete_file.file_format != TFileFormatType::FORMAT_ORC) { return Status::NotSupported("Unsupported Iceberg delete file format {}", @@ -2257,7 +2290,9 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe *reader = std::make_unique( system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, enable_mapping_timestamp_tz, enable_mapping_varbinary, "", preserve_binary_uuid(), - _parquet_profile_cache); + _parquet_profile_cache, + delete_file.__isset.encryption_info ? std::optional(delete_file.encryption_info) + : std::nullopt); } else { *reader = std::make_unique( system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, @@ -2331,25 +2366,32 @@ Status IcebergTableReader::_init_position_delete_rows( IcebergDeleteFileIOContext delete_io_ctx(_runtime_state); for (const auto& delete_file : delete_files) { Status read_status = Status::OK(); - // A position delete file normally references many data files. Cache the complete - // path-to-position map once; caching only the current data file would still rescan the - // shared delete file for every subsequent split. - auto rows_by_data_file = _split_cache->get( - _delete_file_cache_key("iceberg_v2_position_delete_", delete_file.path), - [&]() -> std::shared_ptr { - auto result = - std::make_shared(); - PositionDeleteRowsCollector collector(result.get()); - read_status = _read_position_delete_file(delete_file, delete_scan_params, - &delete_io_ctx, &collector); - if (!read_status.ok()) { - return nullptr; - } - for (auto& [_, rows] : *result) { - std::ranges::sort(rows); - } - return result; - }); + auto load_rows = [&]() -> std::shared_ptr { + auto result = std::make_shared(); + PositionDeleteRowsCollector collector(result.get()); + read_status = _read_position_delete_file(delete_file, delete_scan_params, + &delete_io_ctx, &collector); + if (!read_status.ok()) { + return nullptr; + } + for (auto& [_, rows] : *result) { + std::ranges::sort(rows); + } + return result; + }; + // Keep the cache value alive while another scanner may release the cache's ownership. + std::shared_ptr rows_by_data_file; + if (delete_file.__isset.encryption_info) { + // A warmed path must never authorize another split's missing, wrong or expired key. + // Keep decrypted positions scan-local until encrypted cache identities are defined. + rows_by_data_file = load_rows(); + } else { + // Plaintext position files commonly reference many data files; retain their complete + // path-to-position cache so subsequent splits do not rescan the shared delete file. + rows_by_data_file = _split_cache->get( + _delete_file_cache_key("iceberg_v2_position_delete_", delete_file.path), + load_rows); + } RETURN_IF_ERROR(read_status); DORIS_CHECK(rows_by_data_file != nullptr); const auto rows_it = rows_by_data_file->find(_data_file_path()); @@ -2528,6 +2570,17 @@ Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDe if (!delete_file.__isset.field_ids || delete_file.field_ids.empty()) { return Status::InternalError("Iceberg equality delete file is missing field ids"); } + if (delete_file.__isset.encryption_info) { + EqualityDeleteFilter filter; + RETURN_IF_ERROR( + _load_equality_delete_file(delete_file, scan_params, delete_io_ctx, &filter)); + // Leaving cache_key empty also prevents type-converted equality keys from being shared. + COUNTER_UPDATE(_profile.equality_delete_index_cache_miss_count, 1); + COUNTER_UPDATE(_profile.equality_delete_hash_index_memory, + static_cast(filter.hash_index->memory_usage())); + _equality_delete_filters.push_back(std::move(filter)); + return Status::OK(); + } std::ostringstream cache_key; cache_key << _delete_file_cache_key("iceberg_v2_equality_delete_", delete_file.path); if (scan_params.__isset.current_schema_id) { diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index 90feaac4f8f2b4..19ac9d7acd50c0 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -75,6 +75,12 @@ class IcebergTableReader : public format::TableReader { // Iceberg UUID uses the same 16-byte STRING/VARBINARY carrier in data, defaults and deletes. bool preserve_binary_uuid() const override { return true; } + const TFileEncryptionInfo* file_encryption_info() const override { + return _iceberg_params.has_value() && _iceberg_params->__isset.encryption_info + ? &_iceberg_params->encryption_info + : nullptr; + } + Status validate_file_mapping(const format::TableColumnMapper& mapper) const override; // Iceberg's Parquet reader resolves a data file that carries no field ids at all through the @@ -266,7 +272,8 @@ class IcebergTableReader : public format::TableReader { std::vector delete_key_types; Block delete_block; std::shared_ptr hash_index; - // Split-cache key of this filter, extended by the comparison types for converted copies. + // Empty for encrypted filters: neither the original nor converted plaintext may be cached. + // Other filters extend this split-cache key with the comparison types for converted copies. std::string cache_key; }; std::vector _equality_delete_filters; diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 205dec079d3fd8..f8a1693f9721d6 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -1318,6 +1318,10 @@ Status TableReader::create_next_reader(bool* eos) { Status TableReader::create_file_reader(std::unique_ptr* reader) { DORIS_CHECK(reader != nullptr); + const auto* encryption = file_encryption_info(); + if (encryption != nullptr && _format != FileFormat::PARQUET) { + return Status::NotSupported("File encryption [UNSUPPORTED_FILE_FORMAT]"); + } const bool enable_mapping_timestamp_tz = _scan_params != nullptr && _scan_params->__isset.enable_mapping_timestamp_tz && _scan_params->enable_mapping_timestamp_tz; @@ -1333,7 +1337,8 @@ Status TableReader::create_file_reader(std::unique_ptr* reader) { *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, _global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary, - hive_parquet_time_zone, preserve_binary_uuid(), _parquet_profile_cache); + hive_parquet_time_zone, preserve_binary_uuid(), _parquet_profile_cache, + encryption == nullptr ? std::nullopt : std::optional(*encryption)); return Status::OK(); } if (_format == FileFormat::ORC) { diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index eb402a595e4e38..213dbcd347d991 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -418,6 +418,9 @@ class TableReader { virtual Status create_file_reader(std::unique_ptr* reader); // Some table formats expose UUID as raw bytes rather than the native Doris UUID type. virtual bool preserve_binary_uuid() const { return false; } + + // The descriptor belongs to the current split. File readers copy it before initialization. + virtual const TFileEncryptionInfo* file_encryption_info() const { return nullptr; } virtual TableColumnMappingMode mapping_mode() const { return TableColumnMappingMode::BY_NAME; } virtual void configure_mapper_options(TableColumnMapperOptions*) const {} virtual Status annotate_file_schema(std::vector* file_schema) { diff --git a/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp index d9d7ed27c4ec6e..a5afc20ae9304c 100644 --- a/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -125,9 +126,10 @@ std::shared_ptr int32_array(int32_t value) { return result.ok() ? *result : nullptr; } -void write_mixed_id_position_delete_parquet(const std::string& path, - std::optional row_id = 2147483544, - bool include_child_id = true) { +void write_mixed_id_position_delete_parquet( + const std::string& path, std::optional row_id = 2147483544, + bool include_child_id = true, + const std::shared_ptr<::parquet::FileEncryptionProperties>& encryption = nullptr) { auto id = [](int32_t value) { return arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(value)}); }; @@ -163,6 +165,9 @@ void write_mixed_id_position_delete_parquet(const std::string& path, ASSERT_TRUE(output.ok()) << output.status(); ::parquet::WriterProperties::Builder properties; properties.compression(::parquet::Compression::UNCOMPRESSED); + if (encryption != nullptr) { + properties.encryption(encryption); + } PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, 1, properties.build())); } @@ -250,14 +255,52 @@ format::ColumnDefinition table_column(int32_t id, std::string name, DataTypePtr return column; } -void run_mixed_id_position_delete_test(format::FileFormat file_format, - TFileFormatType::type thrift_format, - const std::string& extension) { +TFileRangeDesc native_position_delete_range(const std::string& path, + TFileFormatType::type thrift_format, + const std::optional& descriptor, + bool encrypted) { + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_path(path); + delete_file.__set_file_format(thrift_format); + if (descriptor.has_value()) { + delete_file.__set_encryption_info(*descriptor); + } + auto range = range_with_delete_file(delete_file); + if (encrypted) { + TFileEncryptionInfo unrelated; + unrelated.__set_version(1); + unrelated.__set_key(std::string(16, 'd')); + unrelated.__set_aad_prefix("unrelated-data-file-aad"); + // A position_deletes scan must select its delete descriptor, not the data-file key. + range.table_format_params.iceberg_params.__set_encryption_info(unrelated); + } + range.__set_path(path); + range.__set_start_offset(0); + range.__set_file_size(static_cast(std::filesystem::file_size(path))); + return range; +} + +void verify_mixed_id_position_delete_row(const Block& block, bool eos) { + ASSERT_FALSE(eos); + ASSERT_EQ(1, block.rows()); + ASSERT_FALSE(struct_child_is_null_at(block, "row", 0, 0)); + EXPECT_EQ(42, struct_int_at(block, "row", 0, 0)); + // Once any file field has an Iceberg id, the id-less sibling is absent rather than name-bound. + EXPECT_TRUE(struct_child_is_null_at(block, "row", 1, 0)); +} + +void run_mixed_id_position_delete_test( + format::FileFormat file_format, TFileFormatType::type thrift_format, + const std::string& extension, + const std::shared_ptr<::parquet::FileEncryptionProperties>& encryption = nullptr, + const std::optional& descriptor = std::nullopt, + bool expect_failure = false) { doris::test::ScopedTempDirectory temp("iceberg_position_delete_mixed_id_" + extension); const auto& test_dir = temp.path(); const auto path = (test_dir / ("delete." + extension)).string(); if (file_format == format::FileFormat::PARQUET) { - write_mixed_id_position_delete_parquet(path); + write_mixed_id_position_delete_parquet(path, 2147483544, true, encryption); } else { write_mixed_id_position_delete_orc(path); } @@ -297,28 +340,29 @@ void run_mixed_id_position_delete_test(format::FileFormat file_format, .file_slot_descs = &slots, }) .ok()); - TIcebergDeleteFileDesc delete_file; - delete_file.__set_content(1); - delete_file.__set_path(path); - delete_file.__set_file_format(thrift_format); - auto range = range_with_delete_file(delete_file); - range.__set_path(path); - range.__set_start_offset(0); - range.__set_file_size(static_cast(std::filesystem::file_size(path))); + auto range = + native_position_delete_range(path, thrift_format, descriptor, encryption != nullptr); format::SplitReadOptions split_options; split_options.current_range = std::move(range); split_options.current_split_format = file_format; - ASSERT_TRUE(reader.prepare_split(split_options).ok()); + auto status = reader.prepare_split(split_options); + if (expect_failure) { + if (status.ok()) { + Block block = make_output_block(slots); + bool eos = false; + status = reader.get_block(&block, &eos); + } + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.to_string().find(std::string(16, 'p')), std::string::npos); + ASSERT_TRUE(reader.close().ok()); + return; + } + ASSERT_TRUE(status.ok()) << status; Block block = make_output_block(slots); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - ASSERT_FALSE(eos); - ASSERT_EQ(1, block.rows()); - ASSERT_FALSE(struct_child_is_null_at(block, "row", 0, 0)); - EXPECT_EQ(42, struct_int_at(block, "row", 0, 0)); - // Once any file field has an Iceberg id, the id-less sibling is absent rather than name-bound. - EXPECT_TRUE(struct_child_is_null_at(block, "row", 1, 0)); + verify_mixed_id_position_delete_row(block, eos); ASSERT_TRUE(reader.close().ok()); } @@ -616,6 +660,62 @@ TEST(IcebergPositionDeleteSysTableV2ReaderTest, OrcRowUsesAnyFieldIdMapping) { run_mixed_id_position_delete_test(format::FileFormat::ORC, TFileFormatType::FORMAT_ORC, "orc"); } +#if __has_include() +TEST(IcebergPositionDeleteSysTableV2ReaderTest, RejectsEncryptedOrcAndPuffinBeforeIo) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("unsupported_encrypted_metadata_delete"); + TFileScanRangeParams params; + std::vector slots; + IcebergPositionDeleteSysTableV2Reader reader; + reader._runtime_state = &state; + reader._scanner_profile = &profile; + reader._scan_params = ¶ms; + reader._file_slot_descs = &slots; + TFileEncryptionInfo info; + info.__set_version(1); + info.__set_key(std::string(16, 'p')); + info.__set_aad_prefix("synthetic-delete-aad"); + for (bool deletion_vector : {false, true}) { + TIcebergDeleteFileDesc descriptor; + descriptor.__set_content(deletion_vector ? 3 : 1); + descriptor.__set_path("missing-delete-file"); + descriptor.__set_file_format(deletion_vector ? TFileFormatType::FORMAT_PARQUET + : TFileFormatType::FORMAT_ORC); + descriptor.__set_encryption_info(info); + reader._current_range = range_with_delete_file(descriptor); + const auto status = reader._init_split(); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("UNSUPPORTED_FILE_FORMAT"), std::string::npos); + EXPECT_EQ(reader._position_reader, nullptr); + } +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, EncryptedParquetRowUsesDeleteFileKeyAndAad) { + TFileEncryptionInfo info; + info.__set_version(1); + info.__set_key(std::string(16, 'p')); + info.__set_aad_prefix("synthetic-position-delete-aad"); + auto encryption = [&]() { + ::parquet::FileEncryptionProperties::Builder builder { + ::arrow::util::SecureString(std::string(info.key))}; + return builder.aad_prefix(info.aad_prefix)->disable_aad_prefix_storage()->build(); + }; + run_mixed_id_position_delete_test(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET, + "encrypted.parquet", encryption(), info); + for (int failure : {0, 1, 2}) { + auto bad = info; + if (failure == 0) { + bad.__set_key(std::string(16, 'd')); + } else { + bad.__set_aad_prefix("different-delete-aad"); + } + run_mixed_id_position_delete_test(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET, + "invalid.parquet", encryption(), + failure == 2 ? std::nullopt : std::optional(bad), true); + } +} +#endif + TEST(IcebergPositionDeleteSysTableV2ReaderTest, ParquetReadsNestedIdlessWrapper) { run_v2_nested_wrapper_position_delete_test(); } diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 930a8a21e74063..04e39deefffbde 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -88,6 +89,7 @@ #include "runtime/runtime_state.h" #include "storage/segment/condition_cache.h" #include "testutil/mock/mock_query_context.h" +#include "testutil/scoped_temp_dir.h" #include "util/debug_points.h" #include "util/defer_op.h" #include "util/hash_util.hpp" @@ -392,9 +394,10 @@ schema::external::TSchema external_schema(int64_t schema_id, return schema; } -void write_iceberg_equality_delete_parquet_file(const std::string& file_path, int32_t field_id, - int32_t value, - const std::string& field_name = "id") { +void write_iceberg_equality_delete_parquet_file( + const std::string& file_path, int32_t field_id, int32_t value, + const std::string& field_name = "id", + std::shared_ptr<::parquet::FileEncryptionProperties> encryption = nullptr) { const auto metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); auto schema = arrow::schema({ @@ -407,6 +410,9 @@ void write_iceberg_equality_delete_parquet_file(const std::string& file_path, in std::shared_ptr out = *file_result; ::parquet::WriterProperties::Builder builder; + if (encryption != nullptr) { + builder.encryption(encryption); + } builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); builder.compression(::parquet::Compression::UNCOMPRESSED); @@ -574,9 +580,10 @@ void write_iceberg_null_equality_delete_parquet_file(const std::string& file_pat builder.build())); } -void write_single_int_parquet_file(const std::string& file_path, const std::string& field_name, - const std::vector& values, - std::optional field_id) { +void write_single_int_parquet_file( + const std::string& file_path, const std::string& field_name, + const std::vector& values, std::optional field_id, + std::shared_ptr<::parquet::FileEncryptionProperties> encryption = nullptr) { auto field = arrow::field(field_name, arrow::int32(), false); if (field_id.has_value()) { field = field->WithMetadata( @@ -589,6 +596,9 @@ void write_single_int_parquet_file(const std::string& file_path, const std::stri std::shared_ptr out = *file_result; ::parquet::WriterProperties::Builder builder; + if (encryption != nullptr) { + builder.encryption(encryption); + } builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); builder.compression(::parquet::Compression::UNCOMPRESSED); @@ -1327,9 +1337,10 @@ void write_iceberg_row_lineage_parquet_file( builder.build())); } -void write_position_delete_parquet_file(const std::string& file_path, - const std::vector& data_file_paths, - const std::vector& positions) { +void write_position_delete_parquet_file( + const std::string& file_path, const std::vector& data_file_paths, + const std::vector& positions, + std::shared_ptr<::parquet::FileEncryptionProperties> encryption = nullptr) { auto schema = arrow::schema({ arrow::field("file_path", arrow::utf8(), false), arrow::field("pos", arrow::int64(), false), @@ -1342,6 +1353,9 @@ void write_position_delete_parquet_file(const std::string& file_path, std::shared_ptr out = *file_result; ::parquet::WriterProperties::Builder builder; + if (encryption != nullptr) { + builder.encryption(encryption); + } builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); builder.compression(::parquet::Compression::UNCOMPRESSED); @@ -9047,5 +9061,272 @@ TEST(IcebergTableReaderTest, DeleteParquetFactoryReusesScannerCounters) { std::filesystem::remove(file_path); } +#if __has_include() +TFileEncryptionInfo synthetic_delete_key(char value) { + TFileEncryptionInfo info; + info.__set_version(1); + info.__set_key(std::string(16, value)); + info.__set_aad_prefix(std::string("synthetic-file-aad-") + value); + return info; +} + +std::shared_ptr<::parquet::FileEncryptionProperties> delete_writer_encryption( + const TFileEncryptionInfo& info) { + ::parquet::FileEncryptionProperties::Builder builder { + ::arrow::util::SecureString(std::string(info.key))}; + return builder.aad_prefix(info.aad_prefix)->disable_aad_prefix_storage()->build(); +} + +Status scan_delete_encryption_split(iceberg::IcebergTableReader* reader, + const SplitReadOptions& split, + const std::vector& columns, + std::vector* rows) { + RETURN_IF_ERROR(reader->prepare_split(split)); + bool eos = false; + while (!eos) { + auto block = build_table_block(columns); + RETURN_IF_ERROR(reader->get_block(&block, &eos)); + for (size_t row = 0; row < block.rows(); ++row) { + rows->push_back((*block.get_by_position(0).column)[row].get()); + } + } + return Status::OK(); +} + +void expect_delete_split_rejection(iceberg::IcebergTableReader* reader, + const SplitReadOptions& split, + const std::vector& columns) { + std::vector ignored; + const auto status = scan_delete_encryption_split(reader, split, columns, &ignored); + EXPECT_FALSE(status.ok()); + for (char key : {'d', 'p', 'e'}) { + EXPECT_EQ(status.to_string().find(synthetic_delete_key(key).key), std::string::npos); + } + // Match scanner recovery: a failed concrete reader must be discarded before another split. + ASSERT_TRUE(reader->abort_split().ok()); +} + +void verify_delete_key_rejections(iceberg::IcebergTableReader* reader, + const SplitReadOptions& split, + const std::vector& columns, + bool position_encrypted, bool equality_encrypted) { + const auto data_key = synthetic_delete_key('d'); + // Warm decoded-delete caches must not authorize a later split with another key. + for (int index : {0, 1}) { + if (!(index == 0 ? position_encrypted : equality_encrypted)) { + continue; + } + for (int failure : {0, 1, 2}) { + auto bad = split; + auto& descriptor = + bad.current_range.table_format_params.iceberg_params.delete_files[index]; + if (failure == 0) { + descriptor.encryption_info.__set_key(data_key.key); + } else if (failure == 1) { + descriptor.__isset.encryption_info = false; + } else { + descriptor.encryption_info.__set_aad_prefix("incorrect-file-aad"); + } + expect_delete_split_rejection(reader, bad, columns); + } + } + if (split.current_range.table_format_params.iceberg_params.__isset.encryption_info) { + for (int failure : {0, 1, 2}) { + auto bad = split; + auto& descriptor = bad.current_range.table_format_params.iceberg_params; + if (failure == 0) { + descriptor.encryption_info.__set_key(synthetic_delete_key('p').key); + } else if (failure == 1) { + descriptor.__isset.encryption_info = false; + } else { + descriptor.encryption_info.__set_aad_prefix("incorrect-data-aad"); + } + expect_delete_split_rejection(reader, bad, columns); + } + } +} + +void verify_encrypted_delete_combination(const std::filesystem::path& directory, + bool data_encrypted, bool position_encrypted, + bool equality_encrypted) { + const auto data_key = synthetic_delete_key('d'); + const auto position_key = synthetic_delete_key('p'); + const auto equality_key = synthetic_delete_key('e'); + const std::string suffix = std::to_string(data_encrypted) + std::to_string(position_encrypted) + + std::to_string(equality_encrypted); + const auto data_path = (directory / ("data-" + suffix + ".parquet")).string(); + const auto position_path = (directory / ("position-" + suffix + ".parquet")).string(); + const auto equality_path = (directory / ("equality-" + suffix + ".parquet")).string(); + write_single_int_parquet_file(data_path, "id", {1, 2, 3, 4, 5, 6, 7, 8}, 1, + data_encrypted ? delete_writer_encryption(data_key) : nullptr); + write_position_delete_parquet_file( + position_path, {data_path, data_path}, {1, 6}, + position_encrypted ? delete_writer_encryption(position_key) : nullptr); + write_iceberg_equality_delete_parquet_file( + equality_path, 1, 5, "id", + equality_encrypted ? delete_writer_encryption(equality_key) : nullptr); + auto position = make_iceberg_position_delete_file(position_path); + auto equality = make_iceberg_equality_delete_file(equality_path, {1}); + if (position_encrypted) { + position.__set_encryption_info(position_key); + } + if (equality_encrypted) { + equality.__set_encryption_info(equality_key); + } + auto split = build_split_options(data_path); + split.current_range.__set_table_format_params( + make_iceberg_table_format_desc(data_path, {position, equality}, 3)); + if (data_encrypted) { + split.current_range.table_format_params.iceberg_params.__set_encryption_info(data_key); + } + auto params = make_local_parquet_scan_params(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("encrypted_delete_scan"); + io::FileReaderStats io_stats; + io::FileCacheStatistics cache_stats; + auto io_ctx = make_io_context(&io_stats, &cache_stats); + const std::vector columns { + make_table_column(1, "id", std::make_shared())}; + ShardedKVCache cache(1); + split.cache = &cache; + iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, columns, ¶ms, io_ctx, &state, &profile); + for (int repeat = 0; repeat < 2; ++repeat) { + std::vector rows; + auto status = scan_delete_encryption_split(&reader, split, columns, &rows); + ASSERT_TRUE(status.ok()) << suffix << ": " << status; + EXPECT_EQ(rows, (std::vector {1, 3, 4, 6, 8})); + } + verify_delete_key_rejections(&reader, split, columns, position_encrypted, equality_encrypted); + std::vector retry_rows; + const auto retry = scan_delete_encryption_split(&reader, split, columns, &retry_rows); + ASSERT_TRUE(retry.ok()) << retry; + EXPECT_EQ(retry_rows, (std::vector {1, 3, 4, 6, 8})); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(IcebergEncryptionTest, DataPositionAndEqualityDeletesUseIndependentKeysAcrossSplits) { + test::ScopedTempDirectory directory("iceberg-encrypted-deletes"); + for (bool data : {false, true}) { + for (bool position : {false, true}) { + for (bool equality : {false, true}) { + verify_encrypted_delete_combination(directory.path(), data, position, equality); + } + } + } +} + +TEST(IcebergEncryptionTest, ConvertedEqualityDeletesDoNotShareDecryptedCacheEntries) { + test::ScopedTempDirectory directory("iceberg-encrypted-converted-deletes"); + const auto data_path = (directory.path() / "data.parquet").string(); + write_nullable_int64_parquet_file(data_path, 1, "id", {1, 2, 3, 4, 5, 6, 7, 8}); + std::vector deletes; + for (int index : {0, 1}) { + const auto key = synthetic_delete_key(index == 0 ? 'e' : 'f'); + const auto path = + (directory.path() / ("equality-" + std::to_string(index) + ".parquet")).string(); + write_iceberg_equality_delete_parquet_file(path, 1, index == 0 ? 2 : 5, "id", + delete_writer_encryption(key)); + auto descriptor = make_iceberg_equality_delete_file(path, {1}); + descriptor.__set_encryption_info(key); + deletes.push_back(std::move(descriptor)); + } + auto params = make_local_parquet_scan_params(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("encrypted_converted_delete_scan"); + io::FileReaderStats io_stats; + io::FileCacheStatistics cache_stats; + auto io_ctx = make_io_context(&io_stats, &cache_stats); + const std::vector columns { + make_table_column(1, "id", std::make_shared())}; + ShardedKVCache cache(1); + auto split = build_split_options(data_path); + split.cache = &cache; + split.current_range.__set_table_format_params( + make_iceberg_table_format_desc(data_path, deletes, 3)); + iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, columns, ¶ms, io_ctx, &state, &profile); + for (int repeat = 0; repeat < 2; ++repeat) { + ASSERT_TRUE(reader.prepare_split(split).ok()); + std::vector rows; + bool eof = false; + while (!eof) { + auto block = build_table_block(columns); + const auto status = reader.get_block(&block, &eof); + ASSERT_TRUE(status.ok()) << status; + for (size_t row = 0; row < block.rows(); ++row) { + rows.push_back((*block.get_by_position(0).column)[row].get()); + } + } + EXPECT_EQ(rows, (std::vector {1, 3, 4, 6, 7, 8})); + } + ASSERT_TRUE(reader.close().ok()); +} + +TEST(IcebergEncryptionTest, UnsupportedEncryptedDeletesFailBeforeFileIo) { + auto params = make_local_parquet_scan_params(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("unsupported_encrypted_delete"); + io::FileReaderStats io_stats; + io::FileCacheStatistics cache_stats; + auto io_ctx = make_io_context(&io_stats, &cache_stats); + const std::vector columns { + make_table_column(1, "id", std::make_shared())}; + iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, columns, ¶ms, io_ctx, &state, &profile); + auto delete_file = make_iceberg_position_delete_file("missing-delete.orc"); + delete_file.__set_file_format(TFileFormatType::FORMAT_ORC); + delete_file.__set_encryption_info(synthetic_delete_key('p')); + IcebergDeleteFileIOContext delete_io_ctx(&state); + std::unique_ptr file_reader; + auto status = + reader._create_delete_file_reader(delete_file, params, &delete_io_ctx, &file_reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("UNSUPPORTED_FILE_FORMAT"), std::string::npos); + EXPECT_EQ(file_reader, nullptr); + delete_file.__set_content(3); + auto desc = make_iceberg_table_format_desc("missing-data.parquet", {delete_file}, 3); + DeleteFileDesc parsed; + bool has_delete = false; + status = reader._parse_deletion_vector_file(desc, &parsed, &has_delete); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("UNSUPPORTED_FILE_FORMAT"), std::string::npos); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(IcebergEncryptionTest, EncryptedAggregatesCannotUseTableLevelRowCounts) { + auto params = make_local_parquet_scan_params(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("encrypted_delete_metadata_count"); + io::FileReaderStats io_stats; + io::FileCacheStatistics cache_stats; + auto io_ctx = make_io_context(&io_stats, &cache_stats); + const std::vector columns { + make_table_column(1, "id", std::make_shared())}; + for (bool encrypted_data : {false, true}) { + iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, columns, ¶ms, io_ctx, &state, &profile); + reader._push_down_agg_type = TPushAggOp::type::COUNT; + reader._push_down_count_columns = std::vector(); + SplitReadOptions split; + split.current_range.__set_path("missing-encrypted-data.parquet"); + split.current_range.__set_file_size(100); + auto position = make_iceberg_position_delete_file("missing-position-delete.parquet"); + auto desc = make_iceberg_table_format_desc("missing-encrypted-data.parquet", {position}, 3); + if (encrypted_data) { + desc.iceberg_params.__set_encryption_info(synthetic_delete_key('d')); + } else { + desc.iceberg_params.delete_files[0].__set_encryption_info(synthetic_delete_key('p')); + } + split.current_range.__set_table_format_params(desc); + set_table_level_row_count(&split, 8); + const auto status = reader.prepare_split(split); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("AGGREGATE_UNAVAILABLE"), std::string::npos); + ASSERT_TRUE(reader.close().ok()); + } +} +#endif + } // namespace } // namespace doris::format diff --git a/docker/thirdparties/docker-compose/iceberg/encryption/IcebergEncryptionFixture.java b/docker/thirdparties/docker-compose/iceberg/encryption/IcebergEncryptionFixture.java index b1ae1c491e2cb9..fb046d8d11fcad 100644 --- a/docker/thirdparties/docker-compose/iceberg/encryption/IcebergEncryptionFixture.java +++ b/docker/thirdparties/docker-compose/iceberg/encryption/IcebergEncryptionFixture.java @@ -125,6 +125,25 @@ public static void main(String[] args) throws Exception { reply(request, 500, "{\"error\":\"Fixture verification failed\"}"); } }); + server.createContext("/native-deletes/reset", request -> { + try (HiveCatalog catalog = fixture.catalog(fixture.properties()); S3FileIO cleanup = new S3FileIO()) { + check("POST".equals(request.getRequestMethod()), "POST required"); + cleanup.initialize(fixture.properties()); + NativeDeleteFixture.reset(catalog, cleanup); + reply(request, 200, "{\"created\":true}"); + } catch (Exception | LinkageError e) { + e.printStackTrace(System.err); + reply(request, 500, "{\"error\":\"Native delete fixture creation failed\"}"); + } + }); + server.createContext("/native-deletes/verify", request -> { + try (HiveCatalog catalog = fixture.catalog(fixture.properties())) { + reply(request, 200, NativeDeleteFixture.verify(catalog)); + } catch (Exception | LinkageError e) { + e.printStackTrace(System.err); + reply(request, 500, "{\"error\":\"Native delete fixture verification failed\"}"); + } + }); // These AWS protocol errors isolate credentials/authorization from storage access and KMS availability. server.createContext("/kms/credentials", request -> reply(request, 400, "{\"__type\":\"UnrecognizedClientException\",\"message\":\"Invalid fixture credentials\"}")); diff --git a/docker/thirdparties/docker-compose/iceberg/encryption/NativeDeleteFixture.java b/docker/thirdparties/docker-compose/iceberg/encryption/NativeDeleteFixture.java new file mode 100644 index 00000000000000..a263b5e721aaa4 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/encryption/NativeDeleteFixture.java @@ -0,0 +1,270 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; + +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.aws.s3.S3FileIO; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.data.parquet.GenericParquetReaders; +import org.apache.iceberg.data.parquet.GenericParquetWriter; +import org.apache.iceberg.deletes.EqualityDeleteWriter; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.deletes.PositionDeleteWriter; +import org.apache.iceberg.encryption.EncryptionUtil; +import org.apache.iceberg.encryption.NativeEncryptionKeyMetadata; +import org.apache.iceberg.hive.HiveCatalog; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.DataWriter; +import org.apache.iceberg.io.DeleteSchemaUtil; +import org.apache.iceberg.parquet.Parquet; +import org.apache.iceberg.types.Types; + +/** Controlled native-reader fixtures. Public synthetic file keys deliberately need no KMS. */ +public final class NativeDeleteFixture { + private static final String DB = "iceberg_encryption_delete_db"; + private static final String WAREHOUSE = "s3://iceberg-encryption/native-deletes"; + private static final Schema SCHEMA = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + + public static void reset(HiveCatalog catalog, S3FileIO cleanup) throws Exception { + Namespace namespace = Namespace.of(DB); + if (!catalog.namespaceExists(namespace)) { + catalog.createNamespace(namespace, Map.of("location", "file:/tmp/" + DB)); + } + for (int version : new int[] {2, 3}) { + for (int mode = 0; mode < 8; mode++) { + String name = "native_deletes_" + mode + "_v" + version; + TableIdentifier identifier = TableIdentifier.of(DB, name); + catalog.dropTable(identifier, false); + String location = WAREHOUSE + "/" + name; + cleanup.deletePrefix(location + "/"); + Table table = catalog.buildTable(identifier, SCHEMA).withLocation(location) + .withProperties(Map.of(TableProperties.FORMAT_VERSION, "2")).create(); + DataFile data = writeData(table, (mode & 1) != 0); + table.newAppend().appendFile(data).commit(); + DeleteFile position = writePosition(table, data, (mode & 2) != 0); + DeleteFile equality = writeEquality(table, (mode & 4) != 0); + table.newRowDelta().addDeletes(position).addDeletes(equality).commit(); + if (version == 3) { + // V3 retains older Parquet position deletes after a V2 snapshot is upgraded. + table.updateProperties().set(TableProperties.FORMAT_VERSION, "3").commit(); + } + } + } + } + + private static String path(Table table, String kind) { + return table.location() + "/data/" + kind + "-" + UUID.randomUUID() + ".parquet"; + } + + private static DataFile writeData(Table table, boolean encrypted) throws Exception { + Parquet.DataWriteBuilder builder = Parquet.writeData(table.io().newOutputFile(path(table, "data"))) + .forTable(table).createWriterFunc(GenericParquetWriter::create); + if (encrypted) { + SyntheticMetadata key = new SyntheticMetadata('d'); + builder.withFileEncryptionKey(key.encryptionKey()).withAADPrefix(key.aadPrefix()).withKeyMetadata(key); + } + DataWriter writer = builder.build(); + try (DataWriter closeable = writer) { + for (int value = 1; value <= 8; value++) { + GenericRecord row = GenericRecord.create(SCHEMA); + row.setField("id", value); + closeable.write(row); + } + } + return writer.toDataFile(); + } + + private static Parquet.DeleteWriteBuilder deleteBuilder(Table table, String kind, boolean encrypted, char key) { + Parquet.DeleteWriteBuilder builder = Parquet.writeDeletes(table.io().newOutputFile(path(table, kind))) + .forTable(table).createWriterFunc(GenericParquetWriter::create); + if (encrypted) { + SyntheticMetadata metadata = new SyntheticMetadata(key); + builder.withFileEncryptionKey(metadata.encryptionKey()).withAADPrefix(metadata.aadPrefix()) + .withKeyMetadata(metadata); + } + return builder; + } + + private static DeleteFile writePosition(Table table, DataFile data, boolean encrypted) throws Exception { + PositionDeleteWriter writer = deleteBuilder(table, "position", encrypted, 'p').rowSchema(null).buildPositionWriter(); + try (PositionDeleteWriter closeable = writer) { + closeable.write(PositionDelete.create().set(data.location(), 1)); + closeable.write(PositionDelete.create().set(data.location(), 6)); + } + return writer.toDeleteFile(); + } + + private static DeleteFile writeEquality(Table table, boolean encrypted) throws Exception { + EqualityDeleteWriter writer = deleteBuilder(table, "equality", encrypted, 'e') + .rowSchema(SCHEMA).equalityFieldIds(1).buildEqualityWriter(); + try (EqualityDeleteWriter closeable = writer) { + GenericRecord row = GenericRecord.create(SCHEMA); + row.setField("id", 5); + closeable.write(row); + } + return writer.toDeleteFile(); + } + + private static List records(Table table, String path, Schema schema, boolean encrypted, char key) { + Parquet.ReadBuilder builder = Parquet.read(table.io().newInputFile(path)).project(schema) + .createReaderFunc(file -> GenericParquetReaders.buildReader(schema, file)); + if (encrypted) { + SyntheticMetadata metadata = new SyntheticMetadata(key); + builder.withFileEncryptionKey(metadata.encryptionKey()).withAADPrefix(metadata.aadPrefix()); + } + List result = new ArrayList<>(); + try (CloseableIterable reader = builder.build()) { + for (Record row : reader) { + result.add(row.copy()); + } + } catch (java.io.IOException e) { + throw new IllegalStateException("Native delete reference read failed", e); + } + return result; + } + + private static boolean matchesKey(ByteBuffer actual, char key, long length) { + // SDK writers add the file length when closing; normalize that field before comparing key/AAD. + return actual != null && EncryptionUtil.setFileLength(actual, length).equals( + EncryptionUtil.setFileLength(new SyntheticMetadata(key).buffer(), length)); + } + + public static String verify(HiveCatalog catalog) throws Exception { + int tables = 0; + for (int version : new int[] {2, 3}) { + for (int mode = 0; mode < 8; mode++) { + Table table = catalog.loadTable(TableIdentifier.of(DB, "native_deletes_" + mode + "_v" + version)); + try (CloseableIterable tasks = table.newScan().planFiles()) { + int files = 0; + for (FileScanTask task : tasks) { + files++; + ByteBuffer dataMetadata = task.file().keyMetadata(); + if ((mode & 1) == 0 ? dataMetadata != null + : !matchesKey(dataMetadata, 'd', task.file().fileSizeInBytes())) { + throw new IllegalStateException("Data key metadata differs from fixture mode " + mode + + ", bytes=" + (dataMetadata == null ? -1 : dataMetadata.remaining())); + } + try (InputStream input = table.io().newInputFile(task.file().location()).newStream()) { + String magic = new String(input.readNBytes(4), StandardCharsets.US_ASCII); + if (!magic.equals((mode & 1) != 0 ? "PARE" : "PAR1")) { + throw new IllegalStateException("Data payload encryption differs from metadata"); + } + } + List data = records(table, task.file().location(), SCHEMA, (mode & 1) != 0, 'd'); + Set positions = new TreeSet<>(); + Set equalities = new TreeSet<>(); + if (task.deletes().size() != 2) { + throw new IllegalStateException("Expected independent position and equality deletes"); + } + for (DeleteFile delete : task.deletes()) { + boolean position = delete.content() == org.apache.iceberg.FileContent.POSITION_DELETES; + boolean encrypted = (mode & (position ? 2 : 4)) != 0; + if (encrypted ? !matchesKey(delete.keyMetadata(), position ? 'p' : 'e', delete.fileSizeInBytes()) + : delete.keyMetadata() != null) { + throw new IllegalStateException("Delete key metadata differs from fixture mode"); + } + try (InputStream input = table.io().newInputFile(delete.location()).newStream()) { + String magic = new String(input.readNBytes(4), StandardCharsets.US_ASCII); + if (!magic.equals(encrypted ? "PARE" : "PAR1")) { + throw new IllegalStateException("Delete payload encryption differs from metadata"); + } + } + Schema schema = position ? DeleteSchemaUtil.pathPosSchema() : SCHEMA; + for (Record row : records(table, delete.location(), schema, encrypted, position ? 'p' : 'e')) { + if (position) { + if (!row.get(0, Object.class).toString().equals(task.file().location())) { + throw new IllegalStateException("Position delete references another data file"); + } + positions.add(row.get(1, Long.class)); + } else { + equalities.add(row.get(0, Integer.class)); + } + } + } + List actual = new ArrayList<>(); + for (int row = 0; row < data.size(); row++) { + int value = data.get(row).get(0, Integer.class); + if (!positions.contains((long) row) && !equalities.contains(value)) { + actual.add(value); + } + } + if (!actual.equals(List.of(1, 3, 4, 6, 8))) { + throw new IllegalStateException("Native delete reference rows differ"); + } + } + if (files != 1) { + throw new IllegalStateException("Expected one data file per fixture"); + } + } + tables++; + } + } + return "{\"tables\":" + tables + ",\"rows\":[1,3,4,6,8],\"independentFileKeys\":true}"; + } + + private static final class SyntheticMetadata implements NativeEncryptionKeyMetadata { + private final byte[] key = new byte[16]; + private final byte[] aad; + + private SyntheticMetadata(char value) { + Arrays.fill(key, (byte) value); + aad = ("synthetic-delete-aad-" + value).getBytes(StandardCharsets.UTF_8); + } + + @Override + public ByteBuffer encryptionKey() { + return ByteBuffer.wrap(key.clone()); + } + + @Override + public ByteBuffer aadPrefix() { + return ByteBuffer.wrap(aad.clone()); + } + + @Override + public ByteBuffer buffer() { + // StandardKeyMetadata V1: Avro bytes(key), optional bytes(AAD), optional long(file length). + // These small public fixtures fit one-byte Avro lengths and intentionally carry no table key chain. + ByteBuffer encoded = ByteBuffer.allocate(21 + aad.length); + encoded.put((byte) 1).put((byte) 32).put(key).put((byte) 2).put((byte) (aad.length * 2)).put(aad).put((byte) 0); + return encoded.flip(); + } + + @Override + public NativeEncryptionKeyMetadata copy() { + return new SyntheticMetadata((char) key[0]); + } + } +} diff --git a/docker/thirdparties/docker-compose/iceberg/encryption/README.md b/docker/thirdparties/docker-compose/iceberg/encryption/README.md new file mode 100644 index 00000000000000..c23a24797b541a --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/encryption/README.md @@ -0,0 +1,39 @@ +# Iceberg encryption reference fixtures + +The isolated service uses Iceberg 1.12.0 and Parquet Java from the checksum-pinned +reference jars prepared by `prepare-jars.sh`. `start.sh` compiles the fixture +classes into the container's temporary directory. Production Doris plugins do +not include these classes. + +The existing `/reset` and `/verify` endpoints exercise HMS metadata encryption +with the test KMS. The native delete endpoints use the same HMS, S3 storage and +fixture service: + +- `POST /native-deletes/reset` replaces only the dedicated + `iceberg_encryption_delete_db` tables and their fixture object prefixes. +- `GET /native-deletes/verify` reads the physical files with the Java SDK and + checks delete application, descriptor keys and actual encrypted footer magic. + +`NativeDeleteFixture` creates all eight combinations of plaintext/encrypted data, +position-delete and equality-delete files. Mode bits 1, 2 and 4 identify those +three file kinds. Their public synthetic AES keys and AAD prefixes are distinct. +Each table starts with values 1 through 8; position deletes remove offsets 1 and +6, and the equality delete removes value 5. The remaining values are +`1, 3, 4, 6, 8`. + +Fixtures exist as V2 tables and V2 snapshots upgraded to V3, which retains their +older Parquet position deletes. Table metadata stays plaintext and carries +synthetic StandardKeyMetadata V1 descriptors directly. This isolates native-file +planning and the last dispatch guard from the separate KMS/metadata-encryption +fixture. It is a controlled test arrangement using public keys, not a production +catalog setup. + +`test_iceberg_encryption_delete_reader` compares plaintext SQL delete results and +position-delete metadata output, and asserts that encrypted descriptors still +stop at the SQL dispatch guard. Metadata-only COUNT rejection is covered separately +by FE and BE unit tests. The SQL suite covers eager/lazy planning, +COUNT pushdown on/off, all file combinations and both format versions. The +physical reader UTs exercise successful decryption, key/AAD failures, repeated +splits, converted equality keys and unsupported encrypted ORC/Puffin descriptors. +Encrypted SQL admission remains closed until transport and cache integration are +complete. diff --git a/docker/thirdparties/docker-compose/iceberg/encryption/start.sh b/docker/thirdparties/docker-compose/iceberg/encryption/start.sh index 9df5c24378f870..ba02770b10483a 100644 --- a/docker/thirdparties/docker-compose/iceberg/encryption/start.sh +++ b/docker/thirdparties/docker-compose/iceberg/encryption/start.sh @@ -20,7 +20,7 @@ set -euo pipefail classpath="/fixtures/jars/iceberg-spark-runtime-4.0_2.13-1.12.0.jar:/fixtures/jars/iceberg-aws-bundle-1.12.0.jar:/opt/spark/jars/*" mkdir -p /tmp/encryption-classes -javac -cp "${classpath}" -d /tmp/encryption-classes /fixtures/IcebergEncryptionFixture.java +javac -cp "${classpath}" -d /tmp/encryption-classes /fixtures/IcebergEncryptionFixture.java /fixtures/NativeDeleteFixture.java exec java -XX:ActiveProcessorCount=4 -Xmx1g -cp "/tmp/encryption-classes:${classpath}" \ IcebergEncryptionFixture thrift://encryption-hms:9083 http://encryption-kms:5000 \ http://minio:9000 /state/kms.properties diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java index a87ab31ddc96f6..0f929834e0cff9 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java @@ -1297,7 +1297,7 @@ private IcebergScanRange buildPositionDeleteRange(PositionDeletesScanTask task, List outputPartitionFields, boolean enableMappingVarbinary, ZoneId zone, UnaryOperator uriNormalizer) { DeleteFile deleteFile = task.file(); - rejectEncryptedDelete(deleteFile); + IcebergFileEncryption encryption = deleteFileEncryption(deleteFile); String originalPath = deleteFile.path().toString(); IcebergScanRange.Builder builder = new IcebergScanRange.Builder() .path(uriNormalizer.apply(originalPath)) @@ -1307,7 +1307,8 @@ private IcebergScanRange buildPositionDeleteRange(PositionDeletesScanTask task, // The split's own scheduling weight, mirroring legacy newPositionDeleteSysTableSplit // (selfSplitWeight = max(length, 1)). .selfSplitWeight(Math.max(task.length(), 1L)) - .targetSplitSize(task.length()); + .targetSplitSize(task.length()) + .encryption(encryption); // DV vs plain position-delete file is decided by content ALONE (BE never looks at the file format): // a puffin DV still travels as FORMAT_PARQUET, exactly as legacy getNativePositionDeleteFileFormat does. @@ -1572,9 +1573,9 @@ private Optional> planCountPushdown(Table table, TableS /** * The capability probe behind {@link #canServeMetadataOnlyCount}: the delete gate of * {@link #planCountPushdown} plus the requirement that the data manifests really carry aggregate row - * counters. Reading only the manifest list keeps this O(manifests) with no data-file enumeration, which is - * what a pre-planning probe can afford; the price is answering {@code false} for the older manifest lists - * that {@code planCountPushdown} still serves through its bounded per-file fallback. Never derives the + * counters. Live data-file key metadata is streamed from the manifests without constructing file tasks + * or opening data files. Older manifest lists without row counters return {@code false}, even though + * {@code planCountPushdown} still serves them through its bounded per-file fallback. Never derives the * count from snapshot summary fields — those are writer-provided hints, not a query result. */ private static boolean canProveCountFromManifests(Table table, TableScan scan, ConnectorSession session) { @@ -1588,17 +1589,42 @@ private static boolean canProveCountFromManifests(Table table, TableScan scan, C if (deleteState == ManifestDeleteState.PRESENT && !netPositionDeletes) { return false; } - OptionalLong positionDeleteRows = deleteState == ManifestDeleteState.NONE - ? OptionalLong.of(0) - : livePositionDeleteRowCount(table, snapshot); + OptionalLong positionDeleteRows; + try { + positionDeleteRows = deleteState == ManifestDeleteState.NONE + ? OptionalLong.of(0) + : livePositionDeleteRowCount(table, snapshot); + } catch (EncryptedCountException e) { + // Capability probing reports unsupported encryption as false; execution must still reject it. + return false; + } if (!positionDeleteRows.isPresent() || (!netPositionDeletes && positionDeleteRows.getAsLong() != 0)) { return false; } - OptionalLong manifestCount = liveRowCountFromManifests(snapshot.dataManifests(table.io())); - return manifestCount.isPresent() - && subtractPositionDeleteRows(manifestCount.getAsLong(), - netPositionDeletes ? positionDeleteRows.getAsLong() : 0).isPresent(); + List dataManifests = snapshot.dataManifests(table.io()); + OptionalLong manifestCount = liveRowCountFromManifests(dataManifests); + if (!manifestCount.isPresent() + || !subtractPositionDeleteRows(manifestCount.getAsLong(), + netPositionDeletes ? positionDeleteRows.getAsLong() : 0).isPresent()) { + return false; + } + // Aggregate counters do not expose per-file encryption. Check every live entry before advertising + // COUNT support, including encrypted files after a plaintext representative; retain no file list. + for (ManifestFile manifest : dataManifests) { + try (ManifestReader reader = ManifestFiles.read(manifest, table.io(), table.specs()) + .select(Collections.singletonList(DataFile.KEY_METADATA.name()))) { + for (DataFile file : reader) { + if (file.keyMetadata() != null) { + return false; + } + } + } catch (IOException e) { + throw new DorisConnectorException( + "Failed to inspect iceberg data manifest " + manifest.path() + ": " + e.getMessage(), e); + } + } + return true; } private Optional> planManifestCountRange(Table table, TableScan scan, @@ -1742,6 +1768,15 @@ private static void checkCountFileEncryption(FileScanTask task) { // Falling back would retain all tasks/ranges before dispatch rejects unsupported encryption. throw new EncryptedCountException(); } + if (task.deletes() != null) { + for (DeleteFile delete : task.deletes()) { + // An encrypted delete also prevents metadata-only COUNT, even for plaintext data. + // Returning a normal fallback would eagerly retain every range before the RPC fence. + if (delete.keyMetadata() != null) { + throw new EncryptedCountException(); + } + } + } } private static final class EncryptedCountException extends DorisConnectorException { @@ -1792,8 +1827,12 @@ private static OptionalLong livePositionDeleteRowCount(Table table, Snapshot sna try (ManifestReader reader = ManifestFiles.readDeleteManifest( manifest, table.io(), table.specs())) { for (DeleteFile delete : reader) { - if (delete.keyMetadata() != null || delete.content() != FileContent.POSITION_DELETES - || delete.recordCount() < 0) { + // Netting delete counts happens before file-task planning. Keep the same early + // encryption rejection here so COUNT cannot escape through the manifest shortcut. + if (delete.keyMetadata() != null) { + throw new EncryptedCountException(); + } + if (delete.content() != FileContent.POSITION_DELETES || delete.recordCount() < 0) { return OptionalLong.empty(); } try { @@ -2031,12 +2070,13 @@ private List buildDeleteFiles(FileScanTask task, return result; } - private static void rejectEncryptedDelete(DeleteFile delete) { - // Plaintext data can reference an independently encrypted delete. Until delete decryption is - // integrated, dropping its key would send BE a plaintext descriptor for encrypted content. - if (delete.keyMetadata() != null) { - throw new UnsupportedOperationException("Iceberg encryption [DELETE_READ_UNAVAILABLE]"); + private static IcebergFileEncryption deleteFileEncryption(DeleteFile delete) { + if (delete.keyMetadata() != null && delete.format() != FileFormat.PARQUET) { + throw new IllegalArgumentException("Iceberg encryption [UNSUPPORTED_FILE_FORMAT]"); } + // Deletes have their own key/AAD even when their data file is plaintext. Decode the + // descriptor independently so a split never substitutes its data key for a delete key. + return IcebergFileEncryption.fromMetadata(delete.keyMetadata()); } /** @@ -2055,7 +2095,7 @@ private static void rejectEncryptedDelete(DeleteFile delete) { * Package-private for direct unit testing. */ IcebergScanRange.DeleteFile convertDelete(DeleteFile delete, UnaryOperator uriNormalizer) { - rejectEncryptedDelete(delete); + IcebergFileEncryption encryption = deleteFileEncryption(delete); String path = uriNormalizer.apply(delete.path().toString()); FileContent content = delete.content(); if (content == FileContent.POSITION_DELETES) { @@ -2070,10 +2110,10 @@ IcebergScanRange.DeleteFile convertDelete(DeleteFile delete, UnaryOperator fieldIds, Long contentOffset, Long contentSizeInBytes, - long fileSize) { + long fileSize, IcebergFileEncryption encryption) { this.path = path; this.content = content; this.fileFormat = fileFormat; @@ -710,13 +714,14 @@ private DeleteFile(String path, int content, TFileFormatType fileFormat, Long po this.contentOffset = contentOffset; this.contentSizeInBytes = contentSizeInBytes; this.fileSize = fileSize; + this.encryption = encryption; } /** A position delete file (content 1): row positions to drop, with optional [lower,upper] bounds. */ public static DeleteFile positionDelete(String path, TFileFormatType fileFormat, Long positionLowerBound, Long positionUpperBound, long fileSize) { return new DeleteFile(path, CONTENT_POSITION_DELETE, fileFormat, - positionLowerBound, positionUpperBound, null, null, null, fileSize); + positionLowerBound, positionUpperBound, null, null, null, fileSize, null); } /** @@ -727,14 +732,22 @@ public static DeleteFile positionDelete(String path, TFileFormatType fileFormat, public static DeleteFile deletionVector(String path, Long positionLowerBound, Long positionUpperBound, long contentOffset, long contentSizeInBytes, long fileSize) { return new DeleteFile(path, CONTENT_DELETION_VECTOR, null, - positionLowerBound, positionUpperBound, null, contentOffset, contentSizeInBytes, fileSize); + positionLowerBound, positionUpperBound, null, contentOffset, contentSizeInBytes, fileSize, null); } /** An equality delete file (content 2): rows equal on {@code fieldIds} are dropped (BE re-projects). */ public static DeleteFile equalityDelete(String path, TFileFormatType fileFormat, List fieldIds, long fileSize) { return new DeleteFile(path, CONTENT_EQUALITY_DELETE, fileFormat, null, null, - fieldIds, null, null, fileSize); + fieldIds, null, null, fileSize, null); + } + + DeleteFile withEncryption(IcebergFileEncryption encryption) { + if (this.encryption == encryption) { + return this; + } + return new DeleteFile(path, content, fileFormat, positionLowerBound, positionUpperBound, + fieldIds, contentOffset, contentSizeInBytes, fileSize, encryption); } int getContent() { @@ -762,6 +775,9 @@ TIcebergDeleteFileDesc toThrift() { if (contentSizeInBytes != null) { desc.setContentSizeInBytes(contentSizeInBytes); } + if (encryption != null) { + desc.setEncryptionInfo(encryption.toThrift()); + } desc.setContent(content); desc.setFileSize(fileSize); return desc; diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java index 1354f60a6b0fdc..10eacd587281e9 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java @@ -211,7 +211,89 @@ public void encryptedFileMetadataReachesEverySplitWithoutAffectingPlainFiles() { } @Test - public void encryptedDeleteFilesAreRejectedBeforeLosingTheirKeys() { + public void encryptedDeleteDescriptorsKeepTheirOwnKeyAndAad() { + Table table = createTable("independent_delete_keys", SCHEMA, PartitionSpec.unpartitioned()); + byte[] metadata = new byte[22]; + metadata[0] = 1; + metadata[1] = 32; + Arrays.fill(metadata, 2, 18, (byte) 13); + metadata[18] = 2; + metadata[19] = 2; + metadata[20] = 9; + for (boolean equality : new boolean[] {false, true}) { + FileMetadata.Builder builder = FileMetadata.deleteFileBuilder(table.spec()); + if (equality) { + builder.ofEqualityDeletes(1); + } else { + builder.ofPositionDeletes(); + } + DeleteFile delete = builder.withPath("s3://bucket/delete.parquet").withFormat("parquet") + .withFileSizeInBytes(100).withRecordCount(1) + .withEncryptionKeyMetadata(ByteBuffer.wrap(metadata)).build(); + TIcebergDeleteFileDesc first = Assertions.assertDoesNotThrow( + () -> provider().convertDelete(delete, UnaryOperator.identity()).toThrift()); + Assertions.assertTrue(first.isSetEncryptionInfo()); + byte[] expected = new byte[16]; + Arrays.fill(expected, (byte) 13); + Assertions.assertArrayEquals(expected, first.getEncryptionInfo().getKey()); + Assertions.assertArrayEquals(new byte[] {9}, first.getEncryptionInfo().getAadPrefix()); + Arrays.fill(first.getEncryptionInfo().getKey(), (byte) 0); + TIcebergDeleteFileDesc second = provider().convertDelete(delete, UnaryOperator.identity()).toThrift(); + Assertions.assertArrayEquals(expected, second.getEncryptionInfo().getKey()); + } + } + + @Test + public void encryptedDeleteFormatsAndPositionDeleteSystemTableAreValidated() { + Table table = createTable("encrypted_delete_formats", SCHEMA, PartitionSpec.unpartitioned()); + byte[] metadata = new byte[22]; + metadata[0] = 1; + metadata[1] = 32; + Arrays.fill(metadata, 2, 18, (byte) 13); + metadata[18] = 2; + metadata[19] = 2; + metadata[20] = 9; + for (FileFormat format : new FileFormat[] {FileFormat.ORC, FileFormat.AVRO, FileFormat.PUFFIN}) { + FileMetadata.Builder builder = FileMetadata.deleteFileBuilder(table.spec()).ofPositionDeletes() + .withPath("s3://bucket/delete").withFormat(format).withFileSizeInBytes(100).withRecordCount(1) + .withEncryptionKeyMetadata(ByteBuffer.wrap(metadata)); + if (format == FileFormat.PUFFIN) { + builder.withContentOffset(16L).withContentSizeInBytes(64L) + .withReferencedDataFile("s3://bucket/data.parquet"); + } + DeleteFile delete = builder.build(); + IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class, + () -> provider().convertDelete(delete, UnaryOperator.identity())); + Assertions.assertEquals("Iceberg encryption [UNSUPPORTED_FILE_FORMAT]", error.getMessage()); + } + DeleteFile delete = FileMetadata.deleteFileBuilder(table.spec()).ofPositionDeletes() + .withPath("s3://bucket/delete.parquet").withFormat(FileFormat.PARQUET) + .withFileSizeInBytes(100).withRecordCount(1) + .withEncryptionKeyMetadata(ByteBuffer.wrap(metadata)).build(); + Table withDeletes = tableWithPositionDelete(delete); + List ranges = planPositionDeletes(withDeletes, Collections.emptyList()); + Assertions.assertFalse(ranges.isEmpty()); + for (ConnectorScanRange range : ranges) { + TTableFormatFileDesc desc = new TTableFormatFileDesc(); + range.populateRangeParams(desc, new TFileRangeDesc()); + TIcebergDeleteFileDesc encoded = desc.getIcebergParams().getDeleteFiles().get(0); + Assertions.assertTrue(encoded.isSetEncryptionInfo()); + byte[] key = new byte[16]; + Arrays.fill(key, (byte) 13); + Assertions.assertArrayEquals(key, encoded.getEncryptionInfo().getKey()); + Assertions.assertArrayEquals(new byte[] {9}, encoded.getEncryptionInfo().getAadPrefix()); + } + IcebergScanPlanProvider countProvider = new IcebergScanPlanProvider( + IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(withDeletes)); + ConnectorSession countSession = new FakeScanSession("UTC", + Collections.singletonMap("ignore_iceberg_dangling_delete", "true")); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> planCount(countProvider, countSession, true)); + Assertions.assertEquals("Iceberg encryption [COUNT_READ_UNAVAILABLE]", error.getMessage()); + } + + @Test + public void malformedEncryptedDeleteMetadataCannotBecomePlaintext() { for (boolean equality : new boolean[] {false, true}) { Table table = createTable("encrypted_deletes_" + equality, SCHEMA, PartitionSpec.unpartitioned()); FileMetadata.Builder builder = FileMetadata.deleteFileBuilder(table.spec()); @@ -225,18 +307,19 @@ public void encryptedDeleteFilesAreRejectedBeforeLosingTheirKeys() { .withEncryptionKeyMetadata(ByteBuffer.wrap(new byte[] {1})).build(); if (!equality) { Table withDeletes = tableWithPositionDelete(delete); - Assertions.assertThrows(UnsupportedOperationException.class, + Assertions.assertThrows(IllegalArgumentException.class, () -> planPositionDeletes(withDeletes, Collections.emptyList())); IcebergScanPlanProvider countProvider = new IcebergScanPlanProvider( IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(withDeletes)); ConnectorSession countSession = new FakeScanSession("UTC", Collections.singletonMap("ignore_iceberg_dangling_delete", "true")); - Assertions.assertThrows(UnsupportedOperationException.class, + DorisConnectorException countError = Assertions.assertThrows(DorisConnectorException.class, () -> planCount(countProvider, countSession, true)); + Assertions.assertEquals("Iceberg encryption [COUNT_READ_UNAVAILABLE]", countError.getMessage()); } - UnsupportedOperationException error = Assertions.assertThrows(UnsupportedOperationException.class, + IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class, () -> provider().convertDelete(delete, UnaryOperator.identity())); - Assertions.assertEquals("Iceberg encryption [DELETE_READ_UNAVAILABLE]", error.getMessage()); + Assertions.assertEquals("Iceberg encryption [INVALID_KEY_METADATA]", error.getMessage()); } } @@ -4072,6 +4155,84 @@ public void metadataOnlyCountCapabilityFollowsTheDeleteGate() { session, new IcebergTableHandle("db1", "t1"), Optional.empty())); } + @ParameterizedTest + @CsvSource({"0,false", "0,true", "1,false", "1,true", "2,false", "2,true", "3,false", "3,true"}) + public void metadataOnlyCountCapabilityChecksIndependentFileEncryption(int encryptedFiles, boolean equality) { + Table table = createTable("count_capability", SCHEMA, PartitionSpec.unpartitioned(), + Collections.singletonMap(TableProperties.FORMAT_VERSION, "2")); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider( + IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); + IcebergTableHandle handle = new IcebergTableHandle("db1", "count_capability"); + Assertions.assertTrue(provider.canServeMetadataOnlyCount(null, handle, Optional.empty())); + + byte[] metadata = new byte[22]; + metadata[0] = 1; + metadata[1] = 32; + Arrays.fill(metadata, 2, 18, (byte) 7); + metadata[18] = 2; + metadata[19] = 2; + metadata[20] = 9; + DataFiles.Builder data = DataFiles.builder(table.spec()).withPath("s3://bucket/data.parquet") + .withFileSizeInBytes(512).withRecordCount(8).withFormat("parquet"); + if ((encryptedFiles & 1) != 0) { + data.withEncryptionKeyMetadata(ByteBuffer.wrap(metadata)); + } + table.newAppend().appendFile(dataFile(table.spec(), "s3://bucket/plain.parquet", 512, null, null)) + .appendFile(data.build()).commit(); + Assertions.assertEquals((encryptedFiles & 1) == 0, + Assertions.assertDoesNotThrow(() -> provider.canServeMetadataOnlyCount(null, handle, Optional.empty())), + "manifest row counters do not establish that every live data file is plaintext"); + + FileMetadata.Builder delete = FileMetadata.deleteFileBuilder(table.spec()) + .withPath("s3://bucket/delete.parquet").withFormat("parquet") + .withFileSizeInBytes(100).withRecordCount(1); + if (equality) { + delete.ofEqualityDeletes(1); + } else { + delete.ofPositionDeletes(); + } + if ((encryptedFiles & 2) != 0) { + delete.withEncryptionKeyMetadata(ByteBuffer.wrap(metadata)); + } + table.newRowDelta().addDeletes(delete.build()).commit(); + for (boolean netPositionDeletes : new boolean[] {false, true}) { + ConnectorSession session = new FakeScanSession("UTC", + Collections.singletonMap("ignore_iceberg_dangling_delete", + Boolean.toString(netPositionDeletes))); + Assertions.assertEquals(encryptedFiles == 0 && !equality && netPositionDeletes, + Assertions.assertDoesNotThrow( + () -> provider.canServeMetadataOnlyCount(session, handle, Optional.empty())), + "unsupported encrypted data/delete combinations must return false, not throw"); + } + } + + @Test + public void metadataOnlyCountCapabilityChecksOnlyLiveFilesInThePinnedSnapshot() { + Table table = createTable("count_capability_snapshots", SCHEMA, PartitionSpec.unpartitioned(), + Collections.singletonMap(TableProperties.FORMAT_VERSION, "2")); + table.newAppend().appendFile( + dataFile(table.spec(), "s3://bucket/plain.parquet", 512, null, null)).commit(); + long plaintextSnapshot = table.currentSnapshot().snapshotId(); + // The probe only needs the presence of key metadata; it must not parse keys or open data files. + DataFile encrypted = DataFiles.builder(table.spec()).withPath("s3://bucket/encrypted.parquet") + .withFormat("parquet").withFileSizeInBytes(512).withRecordCount(4) + .withEncryptionKeyMetadata(ByteBuffer.wrap(new byte[] {1})).build(); + table.newAppend().appendFile(encrypted).commit(); + long encryptedSnapshot = table.currentSnapshot().snapshotId(); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider( + IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); + IcebergTableHandle handle = new IcebergTableHandle("db1", "count_capability_snapshots"); + Assertions.assertTrue(provider.canServeMetadataOnlyCount(null, + handle.withSnapshot(plaintextSnapshot, null, 0L), Optional.empty())); + Assertions.assertFalse(provider.canServeMetadataOnlyCount(null, handle, Optional.empty())); + + table.newDelete().deleteFile(encrypted).commit(); + Assertions.assertTrue(provider.canServeMetadataOnlyCount(null, handle, Optional.empty()), + "deleted entries and historical encrypted snapshots must not block a plaintext count"); + Assertions.assertFalse(provider.canServeMetadataOnlyCount(null, + handle.withSnapshot(encryptedSnapshot, null, 0L), Optional.empty())); + } + @Test public void getScanNodePropertiesUnderPinEmitsFullPinnedSchemaDict() throws Exception { // T07 Option A: under a time-travel pin the field-id dict is built from the FULL pinned schema (covering diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_encryption_delete_reader.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_encryption_delete_reader.out new file mode 100644 index 00000000000000..c6325a823f9ad7 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_encryption_delete_reader.out @@ -0,0 +1,304 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !plain_rows_v2 -- +1 +3 +4 +6 +8 + +-- !plain_count_v2 -- +5 + +-- !plain_positions_0_v2 -- +1 +6 + +-- !plain_position_count_0_v2 -- +2 + +-- !plain_positions_1_v2 -- +1 +6 + +-- !plain_position_count_1_v2 -- +2 + +-- !plain_positions_4_v2 -- +1 +6 + +-- !plain_position_count_4_v2 -- +2 + +-- !plain_positions_5_v2 -- +1 +6 + +-- !plain_position_count_5_v2 -- +2 + +-- !plain_rows_v3 -- +1 +3 +4 +6 +8 + +-- !plain_count_v3 -- +5 + +-- !plain_positions_0_v3 -- +1 +6 + +-- !plain_position_count_0_v3 -- +2 + +-- !plain_positions_1_v3 -- +1 +6 + +-- !plain_position_count_1_v3 -- +2 + +-- !plain_positions_4_v3 -- +1 +6 + +-- !plain_position_count_4_v3 -- +2 + +-- !plain_positions_5_v3 -- +1 +6 + +-- !plain_position_count_5_v3 -- +2 + +-- !plain_rows_v2 -- +1 +3 +4 +6 +8 + +-- !plain_count_v2 -- +5 + +-- !plain_positions_0_v2 -- +1 +6 + +-- !plain_position_count_0_v2 -- +2 + +-- !plain_positions_1_v2 -- +1 +6 + +-- !plain_position_count_1_v2 -- +2 + +-- !plain_positions_4_v2 -- +1 +6 + +-- !plain_position_count_4_v2 -- +2 + +-- !plain_positions_5_v2 -- +1 +6 + +-- !plain_position_count_5_v2 -- +2 + +-- !plain_rows_v3 -- +1 +3 +4 +6 +8 + +-- !plain_count_v3 -- +5 + +-- !plain_positions_0_v3 -- +1 +6 + +-- !plain_position_count_0_v3 -- +2 + +-- !plain_positions_1_v3 -- +1 +6 + +-- !plain_position_count_1_v3 -- +2 + +-- !plain_positions_4_v3 -- +1 +6 + +-- !plain_position_count_4_v3 -- +2 + +-- !plain_positions_5_v3 -- +1 +6 + +-- !plain_position_count_5_v3 -- +2 + +-- !plain_rows_v2 -- +1 +3 +4 +6 +8 + +-- !plain_count_v2 -- +5 + +-- !plain_positions_0_v2 -- +1 +6 + +-- !plain_position_count_0_v2 -- +2 + +-- !plain_positions_1_v2 -- +1 +6 + +-- !plain_position_count_1_v2 -- +2 + +-- !plain_positions_4_v2 -- +1 +6 + +-- !plain_position_count_4_v2 -- +2 + +-- !plain_positions_5_v2 -- +1 +6 + +-- !plain_position_count_5_v2 -- +2 + +-- !plain_rows_v3 -- +1 +3 +4 +6 +8 + +-- !plain_count_v3 -- +5 + +-- !plain_positions_0_v3 -- +1 +6 + +-- !plain_position_count_0_v3 -- +2 + +-- !plain_positions_1_v3 -- +1 +6 + +-- !plain_position_count_1_v3 -- +2 + +-- !plain_positions_4_v3 -- +1 +6 + +-- !plain_position_count_4_v3 -- +2 + +-- !plain_positions_5_v3 -- +1 +6 + +-- !plain_position_count_5_v3 -- +2 + +-- !plain_rows_v2 -- +1 +3 +4 +6 +8 + +-- !plain_count_v2 -- +5 + +-- !plain_positions_0_v2 -- +1 +6 + +-- !plain_position_count_0_v2 -- +2 + +-- !plain_positions_1_v2 -- +1 +6 + +-- !plain_position_count_1_v2 -- +2 + +-- !plain_positions_4_v2 -- +1 +6 + +-- !plain_position_count_4_v2 -- +2 + +-- !plain_positions_5_v2 -- +1 +6 + +-- !plain_position_count_5_v2 -- +2 + +-- !plain_rows_v3 -- +1 +3 +4 +6 +8 + +-- !plain_count_v3 -- +5 + +-- !plain_positions_0_v3 -- +1 +6 + +-- !plain_position_count_0_v3 -- +2 + +-- !plain_positions_1_v3 -- +1 +6 + +-- !plain_position_count_1_v3 -- +2 + +-- !plain_positions_4_v3 -- +1 +6 + +-- !plain_position_count_4_v3 -- +2 + +-- !plain_positions_5_v3 -- +1 +6 + +-- !plain_position_count_5_v3 -- +2 diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_encryption_delete_reader.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_encryption_delete_reader.groovy new file mode 100644 index 00000000000000..b555ba246e4d69 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_encryption_delete_reader.groovy @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonSlurper + +suite("test_iceberg_encryption_delete_reader", "p0,external,nonConcurrent") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableIcebergTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String hms = context.config.otherConfigs.get("iceberg_encryption_metastore_port") + String fixture = context.config.otherConfigs.get("iceberg_encryption_fixture_port") + String minio = context.config.otherConfigs.get("iceberg_minio_port") + def reference = { String path, String method -> + def connection = (HttpURLConnection) new URL("http://${host}:${fixture}/native-deletes/${path}").openConnection() + connection.setRequestMethod(method) + connection.setConnectTimeout(10000) + connection.setReadTimeout(180000) + try { + assertEquals(200, connection.responseCode, "Native delete reference ${path} failed") + return connection.inputStream.withCloseable { new JsonSlurper().parse(it) } + } finally { + connection.disconnect() + } + } + assertTrue(reference("reset", "POST").created) + def verified = reference("verify", "GET") + assertEquals(16, verified.tables) + assertEquals([1, 3, 4, 6, 8], verified.rows) + assertTrue(verified.independentFileKeys) + sql "DROP CATALOG IF EXISTS encryption_delete_regression" + sql """CREATE CATALOG encryption_delete_regression PROPERTIES ( + 'type' = 'iceberg', 'iceberg.catalog.type' = 'hms', + 'hive.metastore.uris' = 'thrift://${host}:${hms}', + 's3.endpoint' = 'http://${host}:${minio}', 's3.region' = 'us-east-1', + 's3.access_key' = 'admin', 's3.secret_key' = 'password', 'use_path_style' = 'true', + 'meta.cache.iceberg.table.enable' = 'false' + )""" + def previous = sql """SELECT @@enable_file_scanner_v2, @@enable_external_table_batch_mode, + @@num_files_in_batch_mode, @@enable_count_push_down_for_external_table""" + try { + sql "SET enable_file_scanner_v2=true" + sql "SET num_files_in_batch_mode=1" + for (String batch : ["false", "true"]) { + sql "SET enable_external_table_batch_mode=${batch}" + for (String pushdown : ["false", "true"]) { + sql "SET enable_count_push_down_for_external_table=${pushdown}" + for (int version : [2, 3]) { + for (int mode = 0; mode < 8; mode++) { + String table = "encryption_delete_regression.iceberg_encryption_delete_db.native_deletes_${mode}_v${version}" + if (mode == 0) { + quickTest("plain_rows_v${version}", "SELECT id FROM ${table} ORDER BY id") + quickTest("plain_count_v${version}", "SELECT COUNT(*) FROM ${table}") + } else { + // The physical reader is tested with real independent keys in UT. SQL + // must still stop before sending any key until the transport stage is ready. + test { + sql "SELECT id FROM ${table} ORDER BY id" + exception "File encryption [READ_UNAVAILABLE]" + } + // Equality deletes prevent metadata-only COUNT here; the query still + // reaches the dispatch fence when aggregate pushdown is requested. + test { + sql "SELECT COUNT(*) FROM ${table}" + exception "File encryption [READ_UNAVAILABLE]" + } + } + if ((mode & 2) == 0) { + // Encrypted data or equality deletes do not encrypt the separate plaintext + // position-delete file exposed by this metadata table. + quickTest("plain_positions_${mode}_v${version}", + "SELECT pos FROM ${table}\$position_deletes ORDER BY pos") + quickTest("plain_position_count_${mode}_v${version}", + "SELECT COUNT(*) FROM ${table}\$position_deletes") + } else { + test { + sql "SELECT pos FROM ${table}\$position_deletes ORDER BY pos" + exception "File encryption [READ_UNAVAILABLE]" + } + test { + sql "SELECT COUNT(*) FROM ${table}\$position_deletes" + exception "File encryption [READ_UNAVAILABLE]" + } + } + } + } + } + } + } finally { + sql "SET enable_file_scanner_v2=${previous[0][0]}" + sql "SET enable_external_table_batch_mode=${previous[0][1]}" + sql "SET num_files_in_batch_mode=${previous[0][2]}" + sql "SET enable_count_push_down_for_external_table=${previous[0][3]}" + } +}