diff --git a/be/src/exec/sink/viceberg_delete_sink.cpp b/be/src/exec/sink/viceberg_delete_sink.cpp index 172fdd28177c62..6d7f189ca2f334 100644 --- a/be/src/exec/sink/viceberg_delete_sink.cpp +++ b/be/src/exec/sink/viceberg_delete_sink.cpp @@ -147,14 +147,16 @@ Status VIcebergDeleteSink::init_properties(ObjectPool* pool) { _compress_type = delete_sink.compress_type; } - // Get output path and table location - if (delete_sink.__isset.output_path) { - _output_path = delete_sink.output_path; - } - - if (delete_sink.__isset.table_location) { - _table_location = delete_sink.table_location; + // Every FE version sends both forms of the data location (see IcebergOutputLocation). Without + // the table's own form the commit could only record the address BE wrote the file at, e.g. + // s3://b/db/t/data/delete_pos_x.parquet for an oss:// table, which other engines resolve with + // the wrong file system. + if (delete_sink.output_path.empty() || delete_sink.table_location.empty()) { + return Status::InternalError( + "Iceberg delete sink must carry output_path and table_location, got '{}' and '{}'", + delete_sink.output_path, delete_sink.table_location); } + _output_location.emplace(delete_sink.output_path, delete_sink.table_location); // Get Hadoop configuration if (delete_sink.__isset.hadoop_config) { @@ -219,7 +221,7 @@ Status VIcebergDeleteSink::open(RuntimeState* state, RuntimeProfile* profile) { LOG(INFO) << fmt::format( "VIcebergDeleteSink opened: delete_type={}, output_path={}, format_version={}", - to_string(_delete_type), _output_path, _format_version); + to_string(_delete_type), _t_sink.iceberg_delete_sink.output_path, _format_version); return Status::OK(); } @@ -461,18 +463,16 @@ Status VIcebergDeleteSink::_collect_position_deletes( Status VIcebergDeleteSink::_write_position_delete_files( const std::map& file_deletions) { + DORIS_CHECK(_output_location.has_value()); constexpr size_t kBatchSize = 4096; for (const auto& [data_file_path, deletion] : file_deletions) { if (deletion.rows_to_delete.isEmpty()) { continue; } - // Generate unique delete file path - std::string delete_file_path = _generate_delete_file_path(data_file_path); - // Create delete file writer auto writer = VIcebergDeleteFileWriterFactory::create_writer( - TFileContent::POSITION_DELETES, delete_file_path, _file_format_type, - _compress_type); + TFileContent::POSITION_DELETES, *_output_location, + _delete_file_name(data_file_path), _file_format_type, _compress_type); // Build column names for position delete std::vector column_names = {"file_path", "pos"}; @@ -486,7 +486,7 @@ Status VIcebergDeleteSink::_write_position_delete_files( writer->open(_state, _state->runtime_profile(), _position_delete_output_expr_ctxs, column_names, _hadoop_conf, _file_type, _broker_addresses); if (writer->file_system() != nullptr) { - _created_files.emplace_back(writer->file_system(), delete_file_path); + _created_files.emplace_back(writer->file_system(), writer->write_path()); } RETURN_IF_ERROR(open_status); @@ -523,7 +523,7 @@ Status VIcebergDeleteSink::_write_position_delete_files( _delete_file_count++; VLOG(1) << fmt::format("Written position delete file: path={}, rows={}, referenced_file={}", - delete_file_path, commit_data.row_count, data_file_path); + writer->write_path(), commit_data.row_count, data_file_path); } return Status::OK(); @@ -623,6 +623,7 @@ std::string VIcebergDeleteSink::_get_file_extension() const { Status VIcebergDeleteSink::_write_deletion_vector_files( const std::map& file_deletions) { + DORIS_CHECK(_output_location.has_value()); std::vector blobs; for (const auto& [data_file_path, deletion] : file_deletions) { if (deletion.rows_to_delete.isEmpty()) { @@ -667,13 +668,15 @@ Status VIcebergDeleteSink::_write_deletion_vector_files( return Status::OK(); } - std::string puffin_path = _generate_puffin_file_path(); + const std::string puffin_file_name = _puffin_file_name(); int64_t puffin_file_size = 0; - RETURN_IF_ERROR(_write_puffin_file(puffin_path, &blobs, &puffin_file_size)); + RETURN_IF_ERROR(_write_puffin_file(_output_location->write_path(puffin_file_name), &blobs, + &puffin_file_size)); + const std::string commit_path = _output_location->commit_path(puffin_file_name); for (const auto& blob : blobs) { TIcebergCommitData commit_data; - commit_data.__set_file_path(puffin_path); + commit_data.__set_file_path(commit_path); commit_data.__set_row_count(blob.merged_count); commit_data.__set_affected_rows(blob.delete_count); commit_data.__set_file_size(puffin_file_size); @@ -783,37 +786,15 @@ std::string VIcebergDeleteSink::_build_puffin_footer_json( return {buffer.GetString(), buffer.GetSize()}; } -std::string VIcebergDeleteSink::_generate_delete_file_path( - const std::string& referenced_data_file) { - // Generate unique delete file name using UUID - std::string uuid = generate_uuid_string(); - std::string file_name; - - std::string file_extension = _get_file_extension(); - file_name = - fmt::format("delete_pos_{}_{}{}", uuid, - std::hash {}(referenced_data_file) % 10000000, file_extension); - - // Combine with output path or table location - std::string base_path = _output_path.empty() ? _table_location : _output_path; - - // Ensure base path ends with / - if (!base_path.empty() && base_path.back() != '/') { - base_path += '/'; - } - - // Delete files are data files in Iceberg, write under data location - return fmt::format("{}{}", base_path, file_name); +std::string VIcebergDeleteSink::_delete_file_name(const std::string& referenced_data_file) const { + // Delete files live in the data location itself, next to the partition directories. + return fmt::format("delete_pos_{}_{}{}", generate_uuid_string(), + std::hash {}(referenced_data_file) % 10000000, + _get_file_extension()); } -std::string VIcebergDeleteSink::_generate_puffin_file_path() { - std::string uuid = generate_uuid_string(); - std::string file_name = fmt::format("delete_dv_{}.puffin", uuid); - std::string base_path = _output_path.empty() ? _table_location : _output_path; - if (!base_path.empty() && base_path.back() != '/') { - base_path += '/'; - } - return fmt::format("{}{}", base_path, file_name); +std::string VIcebergDeleteSink::_puffin_file_name() const { + return fmt::format("delete_dv_{}.puffin", generate_uuid_string()); } } // namespace doris diff --git a/be/src/exec/sink/viceberg_delete_sink.h b/be/src/exec/sink/viceberg_delete_sink.h index 55698ae0404b14..383fb7ece6e7f4 100644 --- a/be/src/exec/sink/viceberg_delete_sink.h +++ b/be/src/exec/sink/viceberg_delete_sink.h @@ -24,12 +24,14 @@ #include #include #include +#include #include #include #include "common/status.h" #include "core/block/block.h" #include "exec/sink/writer/async_result_writer.h" +#include "exec/sink/writer/iceberg/iceberg_output_location.h" #include "exec/sink/writer/iceberg/viceberg_delete_file_writer.h" #include "exprs/vexpr_fwd.h" #include "roaring/roaring64map.hh" @@ -114,12 +116,9 @@ class VIcebergDeleteSink final : public AsyncResultWriter { std::string _build_puffin_footer_json(const std::vector& blobs); - std::string _generate_puffin_file_path(); - - /** - * Generate unique delete file path - */ - std::string _generate_delete_file_path(const std::string& referenced_data_file = ""); + // Unique file names, relative to the data location (_output_location). + std::string _puffin_file_name() const; + std::string _delete_file_name(const std::string& referenced_data_file) const; /** * Get $row_id column index from block @@ -162,9 +161,8 @@ class VIcebergDeleteSink final : public AsyncResultWriter { TFileFormatType::type _file_format_type = TFileFormatType::FORMAT_PARQUET; TFileCompressType::type _compress_type = TFileCompressType::SNAPPYBLOCK; - // Output directory for delete files - std::string _output_path; - std::string _table_location; + // The data location delete files are written to and committed under, set by init_properties(). + std::optional _output_location; TFileType::type _file_type = TFileType::FILE_HDFS; std::vector _broker_addresses; diff --git a/be/src/exec/sink/writer/iceberg/iceberg_output_location.h b/be/src/exec/sink/writer/iceberg/iceberg_output_location.h new file mode 100644 index 00000000000000..58982f73156fd0 --- /dev/null +++ b/be/src/exec/sink/writer/iceberg/iceberg_output_location.h @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/check.h" + +namespace doris { + +// The directory an Iceberg writer puts its files in, in both forms of the same location. BE +// creates, writes and deletes a file under write_dir, the form BE addresses the storage with. +// Iceberg metadata records the file under commit_dir, the table's own form, which every engine +// reading the table resolves. FE sends both: TIcebergTableSink output_path / original_output_path +// and TIcebergDeleteSink output_path / table_location. Example: an OSS table with data location +// oss://b/db/t/data reaches BE as write_dir s3://b/db/t/data and commit_dir oss://b/db/t/data, so +// delete_pos_1.parquet is written to s3://b/db/t/data/delete_pos_1.parquet and committed as +// oss://b/db/t/data/delete_pos_1.parquet. +// +// Every file a writer reports takes both of its paths from one IcebergOutputLocation, so it +// cannot commit the form it wrote: delete files used to be committed as s3://..., and other +// engines resolved them with a file system the table is not on. +class IcebergOutputLocation { +public: + IcebergOutputLocation(std::string write_dir, std::string commit_dir) + : _write_dir(std::move(write_dir)), _commit_dir(std::move(commit_dir)) { + DORIS_CHECK(!_write_dir.empty()); + DORIS_CHECK(!_commit_dir.empty()); + } + + // The same subdirectory in both forms, e.g. the partition directory "p=1/q=2". + IcebergOutputLocation subdirectory(std::string_view relative_dir) const { + DORIS_CHECK(!relative_dir.empty()); + return {_join(_write_dir, relative_dir), _join(_commit_dir, relative_dir)}; + } + + // The path BE writes file_name at. + std::string write_path(std::string_view file_name) const { + return _join(_write_dir, file_name); + } + + // The path Iceberg metadata records file_name at. + std::string commit_path(std::string_view file_name) const { + return _join(_commit_dir, file_name); + } + +private: + // FE takes write.data.path verbatim, so a directory may end with '/'. Both forms join with + // exactly one separator, as Iceberg's own location provider does: oss://b/t/data and + // oss://b/t/data/ both join "p=1" into oss://b/t/data/p=1. + static std::string _join(const std::string& dir, std::string_view name) { + std::string path = dir; + if (path.back() != '/') { + path.push_back('/'); + } + path.append(name); + return path; + } + + std::string _write_dir; + std::string _commit_dir; +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp index 8ad7751e1ed19d..b6a0bfa7171de3 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp @@ -43,11 +43,13 @@ std::unique_ptr build_position_delete_schema() { } VIcebergDeleteFileWriter::VIcebergDeleteFileWriter(TFileContent::type delete_type, - const std::string& output_path, + const IcebergOutputLocation& location, + const std::string& file_name, TFileFormatType::type file_format, TFileCompressType::type compress_type) : _delete_type(delete_type), - _output_path(output_path), + _write_path(location.write_path(file_name)), + _commit_path(location.commit_path(file_name)), _file_format(file_format), _compress_type(compress_type) {} @@ -73,7 +75,7 @@ Status VIcebergDeleteFileWriter::open(RuntimeState* state, RuntimeProfile* profi fs_properties.broker_addresses = &broker_addresses; } - io::FileDescription file_description = {.path = _output_path, .fs_name {}}; + io::FileDescription file_description = {.path = _write_path, .fs_name {}}; _fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description)); io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false}; @@ -147,7 +149,7 @@ Status VIcebergDeleteFileWriter::close(TIcebergCommitData& commit_data) { } // Fill commit data (use __set_ to mark optional fields as present) - commit_data.__set_file_path(_output_path); + commit_data.__set_file_path(_commit_path); commit_data.__set_row_count(_written_rows); commit_data.__set_affected_rows(_written_rows); commit_data.__set_file_size(_file_size); @@ -168,9 +170,10 @@ Status VIcebergDeleteFileWriter::close(TIcebergCommitData& commit_data) { // Factory method std::unique_ptr VIcebergDeleteFileWriterFactory::create_writer( - TFileContent::type delete_type, const std::string& output_path, - TFileFormatType::type file_format, TFileCompressType::type compress_type) { - return std::make_unique(delete_type, output_path, file_format, + TFileContent::type delete_type, const IcebergOutputLocation& location, + const std::string& file_name, TFileFormatType::type file_format, + TFileCompressType::type compress_type) { + return std::make_unique(delete_type, location, file_name, file_format, compress_type); } diff --git a/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h index 4ceb7080a3c244..b216b3ba16f678 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h @@ -23,6 +23,7 @@ #include "common/status.h" #include "core/block/block.h" +#include "exec/sink/writer/iceberg/iceberg_output_location.h" #include "exprs/vexpr_fwd.h" #include "gen_cpp/DataSinks_types.h" #include "io/fs/file_writer.h" @@ -50,8 +51,10 @@ class VFileFormatTransformer; */ class VIcebergDeleteFileWriter { public: - VIcebergDeleteFileWriter(TFileContent::type delete_type, const std::string& output_path, - TFileFormatType::type file_format, + // Writes file_name under location; the commit data reports the file under the table's own form + // of that directory, not the form it was written at. + VIcebergDeleteFileWriter(TFileContent::type delete_type, const IcebergOutputLocation& location, + const std::string& file_name, TFileFormatType::type file_format, TFileCompressType::type compress_type); ~VIcebergDeleteFileWriter(); @@ -96,9 +99,13 @@ class VIcebergDeleteFileWriter { std::shared_ptr file_system() const { return _fs; } + // The path the file is written at, which is what cleanup must delete. + const std::string& write_path() const { return _write_path; } + private: TFileContent::type _delete_type; - std::string _output_path; + std::string _write_path; + std::string _commit_path; TFileFormatType::type _file_format; TFileCompressType::type _compress_type = TFileCompressType::SNAPPYBLOCK; @@ -122,8 +129,9 @@ class VIcebergDeleteFileWriter { class VIcebergDeleteFileWriterFactory { public: static std::unique_ptr create_writer( - TFileContent::type delete_type, const std::string& output_path, - TFileFormatType::type file_format, TFileCompressType::type compress_type); + TFileContent::type delete_type, const IcebergOutputLocation& location, + const std::string& file_name, TFileFormatType::type file_format, + TFileCompressType::type compress_type); }; } // namespace doris diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp index e1d2de9c439652..95ef5c960357ce 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp @@ -63,7 +63,7 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil if (!_write_info.broker_addresses.empty()) { fs_properties.broker_addresses = &(_write_info.broker_addresses); } - _path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name()); + _path = _write_info.location.write_path(_get_target_file_name()); io::FileDescription file_description = {.path = _path, .fs_name {}}; _fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description)); io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false}; @@ -165,8 +165,7 @@ Status VIcebergPartitionWriter::_build_iceberg_commit_data(TIcebergCommitData* c DCHECK(commit_data != nullptr); DCHECK(_file_format_transformer != nullptr); - commit_data->__set_file_path( - fmt::format("{}/{}", _write_info.original_write_path, _get_target_file_name())); + commit_data->__set_file_path(_write_info.location.commit_path(_get_target_file_name())); commit_data->__set_row_count(_row_count); commit_data->__set_affected_rows(_row_count); commit_data->__set_file_size(_file_format_transformer->written_len()); diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp index 71c5c1c931da7c..26199b68e5e362 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -56,6 +56,18 @@ Status VIcebergTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { _state = state; _operator_profile = profile; + // Every FE version sends both forms of the data location (see IcebergOutputLocation). A plan + // without the table's own form leaves no path to commit the files under: the commit used to + // record a relative path such as /p=1/xxx.parquet that no engine can open. + const auto& iceberg_sink = _t_sink.iceberg_table_sink; + if (iceberg_sink.output_path.empty() || iceberg_sink.original_output_path.empty()) { + return Status::InternalError( + "Iceberg table sink must carry output_path and original_output_path, got '{}' " + "and '{}'", + iceberg_sink.output_path, iceberg_sink.original_output_path); + } + _output_location.emplace(iceberg_sink.output_path, iceberg_sink.original_output_path); + // Get target file size from query options // If value is 0 or not set, use config::iceberg_sink_max_file_size _target_file_size_bytes = config::iceberg_sink_max_file_size; @@ -829,47 +841,29 @@ std::vector VIcebergTableWriter::_partition_values( std::shared_ptr VIcebergTableWriter::_create_partition_writer( Block* transformed_block, int position, const std::string* file_name, int file_name_index) { + DORIS_CHECK(_output_location.has_value()); auto& iceberg_table_sink = _t_sink.iceberg_table_sink; std::vector partition_values; - const std::string& output_path = iceberg_table_sink.output_path; - std::string write_path; - std::string original_write_path; - std::string target_path; + // Relative to the data location, e.g. "year=2023/month=12"; empty when the file goes to the + // data location itself. + std::string partition_path; // Case 1: Full static partition - use pre-computed static partition path and values if (_is_full_static_partition) { partition_values = _static_partition_value_list; - if (!_static_partition_path.empty()) { - original_write_path = fmt::format("{}/{}", iceberg_table_sink.original_output_path, - _static_partition_path); - target_path = fmt::format("{}/{}", output_path, _static_partition_path); - write_path = fmt::format("{}/{}", output_path, _static_partition_path); - } else { - original_write_path = iceberg_table_sink.original_output_path; - target_path = output_path; - write_path = output_path; - } + partition_path = _static_partition_path; } else if (transformed_block != nullptr) { // Case 2: Dynamic partition or Hybrid mode (partial static + partial dynamic) // _partition_to_path and _partition_values already handle hybrid mode internally IcebergPartitionData partition_data = _get_partition_data(transformed_block, position); - std::string partition_path = _partition_to_path(partition_data); + partition_path = _partition_to_path(partition_data); partition_values = _partition_values(partition_data); - original_write_path = - fmt::format("{}/{}", iceberg_table_sink.original_output_path, partition_path); - target_path = fmt::format("{}/{}", output_path, partition_path); - write_path = fmt::format("{}/{}", output_path, partition_path); - } else { - // Case 3: Non-partitioned table - original_write_path = iceberg_table_sink.original_output_path; - target_path = output_path; - write_path = output_path; } + // Case 3: Non-partitioned table, written to the data location itself IPartitionWriterBase::WriteInfo write_info = { - .write_path = std::move(write_path), - .original_write_path = std::move(original_write_path), - .target_path = std::move(target_path), + .location = partition_path.empty() ? *_output_location + : _output_location->subdirectory(partition_path), .file_type = iceberg_table_sink.file_type, .broker_addresses = {}}; if (iceberg_table_sink.__isset.broker_addresses) { diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h index 5baf351c72c9ed..2f0a5dba51921d 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h @@ -20,10 +20,13 @@ #include #include +#include + #include "common/atomic_shared_ptr.h" #include "common/status.h" #include "core/block/block.h" #include "core/column/column.h" +#include "exec/sink/writer/iceberg/iceberg_output_location.h" #include "exec/sink/writer/iceberg/partition_data.h" #include "exec/sink/writer/iceberg/partition_transformers.h" #include "exprs/vexpr_fwd.h" @@ -173,6 +176,8 @@ class VIcebergTableWriter { TDataSink _t_sink; RuntimeState* _state = nullptr; RuntimeProfile* _operator_profile = nullptr; + // The table's data location, set by open(); partition writers write under its subdirectories. + std::optional _output_location; // Target file size in bytes for controlling when to split files int64_t _target_file_size_bytes = 0; diff --git a/be/src/exec/sink/writer/iceberg/vpartition_writer_base.h b/be/src/exec/sink/writer/iceberg/vpartition_writer_base.h index dc70f64679597a..0c1980ca252ba9 100644 --- a/be/src/exec/sink/writer/iceberg/vpartition_writer_base.h +++ b/be/src/exec/sink/writer/iceberg/vpartition_writer_base.h @@ -20,6 +20,7 @@ #include #include "common/status.h" +#include "exec/sink/writer/iceberg/iceberg_output_location.h" #include "runtime/descriptors.h" namespace doris { @@ -34,9 +35,9 @@ class RuntimeProfile; class IPartitionWriterBase { public: struct WriteInfo { - std::string write_path; - std::string original_write_path; - std::string target_path; + // The directory of the partition this writer writes, in the form BE writes the file in + // and the form the commit records it in. + IcebergOutputLocation location; TFileType::type file_type; std::vector broker_addresses; }; diff --git a/be/test/exec/sink/viceberg_delete_sink_test.cpp b/be/test/exec/sink/viceberg_delete_sink_test.cpp index 7faa77ed702c68..a8e1c7cf0cdc22 100644 --- a/be/test/exec/sink/viceberg_delete_sink_test.cpp +++ b/be/test/exec/sink/viceberg_delete_sink_test.cpp @@ -17,7 +17,10 @@ #include "exec/sink/viceberg_delete_sink.h" +#include +#include #include +#include #include #include @@ -36,7 +39,9 @@ #include "format/table/deletion_vector.h" #include "gen_cpp/DataSinks_types.h" #include "gen_cpp/Types_types.h" +#include "runtime/exec_env.h" #include "runtime/runtime_state.h" +#include "util/defer_op.h" #include "util/uid_util.h" namespace doris { @@ -459,26 +464,57 @@ TEST_F(VIcebergDeleteSinkTest, TestBuildPositionDeleteBlock) { ASSERT_EQ(40, pos_column->get_element(3)); } -TEST_F(VIcebergDeleteSinkTest, TestGenerateDeleteFilePath) { +TEST_F(VIcebergDeleteSinkTest, TestDeleteFileNames) { VExprContextSPtrs output_exprs; auto sink = std::make_shared(_t_data_sink, output_exprs, nullptr, nullptr); ObjectPool pool; - Status status = sink->init_properties(&pool); - ASSERT_TRUE(status.ok()); + ASSERT_TRUE(sink->init_properties(&pool).ok()); - std::string data_file_path = "data/file1.parquet"; - std::string delete_file_path = sink->_generate_delete_file_path(data_file_path); + // Names are relative to the data location; the location supplies both forms of the directory. + std::string delete_file_name = sink->_delete_file_name("data/file1.parquet"); + EXPECT_TRUE(delete_file_name.starts_with("delete_pos_")) << delete_file_name; + EXPECT_TRUE(delete_file_name.ends_with(".snappy.parquet")) << delete_file_name; + EXPECT_EQ(std::string::npos, delete_file_name.find('/')); + EXPECT_NE(delete_file_name, sink->_delete_file_name("data/file1.parquet")); + + std::string puffin_file_name = sink->_puffin_file_name(); + EXPECT_TRUE(puffin_file_name.starts_with("delete_dv_")) << puffin_file_name; + EXPECT_TRUE(puffin_file_name.ends_with(".puffin")) << puffin_file_name; + EXPECT_EQ(std::string::npos, puffin_file_name.find('/')); +} - // Verify the path format - ASSERT_FALSE(delete_file_path.empty()); - const auto& delete_sink = _t_data_sink.iceberg_delete_sink; - std::string expected_base = - delete_sink.__isset.output_path ? delete_sink.output_path : delete_sink.table_location; - if (!expected_base.empty() && expected_base.back() != '/') { - expected_base += '/'; - } - ASSERT_TRUE(delete_file_path.rfind(expected_base, 0) == 0); - ASSERT_NE(std::string::npos, delete_file_path.find("delete_pos_")); +TEST_F(VIcebergDeleteSinkTest, TestInitPropertiesTakesBothFormsOfTheDataLocation) { + // FE writes through the normalized output_path but the table records its own location. + TDataSink t_data_sink = _t_data_sink; + t_data_sink.iceberg_delete_sink.__set_output_path("s3://b/db/t/data"); + t_data_sink.iceberg_delete_sink.__set_table_location("oss://b/db/t/data/"); + VExprContextSPtrs output_exprs; + auto sink = std::make_shared(t_data_sink, output_exprs, nullptr, nullptr); + ObjectPool pool; + ASSERT_TRUE(sink->init_properties(&pool).ok()); + ASSERT_TRUE(sink->_output_location.has_value()); + EXPECT_EQ("s3://b/db/t/data/delete_dv_1.puffin", + sink->_output_location->write_path("delete_dv_1.puffin")); + EXPECT_EQ("oss://b/db/t/data/delete_dv_1.puffin", + sink->_output_location->commit_path("delete_dv_1.puffin")); + + // Without the table's own form the commit could only record the s3:// address BE wrote. + t_data_sink.iceberg_delete_sink.__isset.table_location = false; + t_data_sink.iceberg_delete_sink.table_location.clear(); + auto no_table_location = + std::make_shared(t_data_sink, output_exprs, nullptr, nullptr); + Status status = no_table_location->init_properties(&pool); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(std::string::npos, status.to_string().find("table_location")); + + t_data_sink.iceberg_delete_sink.__set_table_location("oss://b/db/t/data/"); + t_data_sink.iceberg_delete_sink.__isset.output_path = false; + t_data_sink.iceberg_delete_sink.output_path.clear(); + auto no_output_path = + std::make_shared(t_data_sink, output_exprs, nullptr, nullptr); + status = no_output_path->init_properties(&pool); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(std::string::npos, status.to_string().find("output_path")); } TEST_F(VIcebergDeleteSinkTest, TestUnsupportedDeleteType) { @@ -521,12 +557,88 @@ TEST_F(VIcebergDeleteSinkTest, TestValidateDeletionVectorContentSize) { ASSERT_TRUE(unsupported_status.is()) << unsupported_status; } +TEST_F(VIcebergDeleteSinkTest, TestWritePositionDeletesCommitsTableLocation) { + // The Parquet writer's dictionary encoder allocates from ExecEnv's Arrow pool, which this + // fixture does not otherwise initialize. + auto* env = ExecEnv::GetInstance(); + auto* previous_pool = env->_arrow_memory_pool; + env->_arrow_memory_pool = arrow::default_memory_pool(); + Defer restore_pool([&] { env->_arrow_memory_pool = previous_pool; }); + + std::filesystem::path temp_dir = std::filesystem::temp_directory_path() / + ("iceberg_delete_sink_test_" + generate_uuid_string()); + ASSERT_TRUE(std::filesystem::create_directories(temp_dir)); + + // Write locally, but record the table's original URI in the commit metadata. + const std::string table_location = "oss://bucket/warehouse/table/data/"; + const std::string data_file_path = table_location + "file1.parquet"; + TDataSink t_data_sink = build_local_delete_sink(temp_dir.string(), 2); + t_data_sink.iceberg_delete_sink.__set_table_location(table_location); + RuntimeState state; + VExprContextSPtrs output_exprs; + auto sink = std::make_shared(t_data_sink, output_exprs, nullptr, nullptr); + ObjectPool pool; + ASSERT_TRUE(sink->init_properties(&pool).ok()); + sink->_state = &state; + + std::map file_deletions; + auto [it, inserted] = + file_deletions.emplace(data_file_path, IcebergFileDeletion(1, "[\"p=1\"]")); + ASSERT_TRUE(inserted); + it->second.rows_to_delete.add(uint32_t {10}); + it->second.rows_to_delete.add(uint32_t {20}); + + const auto status = sink->_write_position_delete_files(file_deletions); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(1, sink->_commit_data_list.size()); + ASSERT_EQ(1, sink->_created_files.size()); + ASSERT_EQ(1, sink->_delete_file_count); + + const std::filesystem::path write_path = sink->_created_files[0].second; + ASSERT_EQ(temp_dir, write_path.parent_path()); + ASSERT_TRUE(std::filesystem::exists(write_path)); + const auto& commit_data = sink->_commit_data_list[0]; + EXPECT_EQ(table_location + write_path.filename().string(), commit_data.file_path); + EXPECT_EQ(data_file_path, commit_data.referenced_data_file_path); + EXPECT_EQ(TFileContent::POSITION_DELETES, commit_data.file_content); + EXPECT_EQ(2, commit_data.row_count); + EXPECT_EQ(2, commit_data.affected_rows); + EXPECT_EQ(1, commit_data.partition_spec_id); + EXPECT_EQ("[\"p=1\"]", commit_data.partition_data_json); + EXPECT_EQ(std::filesystem::file_size(write_path), commit_data.file_size); + + // The referenced data-file URI inside the position-delete file must stay unchanged too. + auto input = arrow::io::ReadableFile::Open(write_path.string()); + ASSERT_TRUE(input.ok()) << input.status(); + auto reader = ::parquet::arrow::OpenFile(*input, arrow::default_memory_pool()); + ASSERT_TRUE(reader.ok()) << reader.status(); + auto table = (*reader)->ReadTable(); + ASSERT_TRUE(table.ok()) << table.status(); + ASSERT_EQ(2, (*table)->num_rows()); + ASSERT_EQ(2, (*table)->num_columns()); + for (int64_t row = 0; row < 2; ++row) { + auto file_path = (*table)->column(0)->GetScalar(row); + ASSERT_TRUE(file_path.ok()) << file_path.status(); + EXPECT_EQ(data_file_path, (*file_path)->ToString()); + auto position = (*table)->column(1)->GetScalar(row); + ASSERT_TRUE(position.ok()) << position.status(); + EXPECT_EQ(std::to_string((row + 1) * 10), (*position)->ToString()); + } + + std::filesystem::remove_all(temp_dir); +} + TEST_F(VIcebergDeleteSinkTest, TestWriteDeletionVectorsToSingleSharedPuffin) { std::filesystem::path temp_dir = std::filesystem::temp_directory_path() / ("iceberg_delete_sink_test_" + generate_uuid_string()); ASSERT_TRUE(std::filesystem::create_directories(temp_dir)); - TDataSink t_data_sink = build_local_delete_sink(temp_dir.string(), 3); + const std::string table_location = + "abfss://container@account.dfs.core.windows.net/warehouse/table/data"; + const std::string first_data_file = table_location + "/file1.parquet"; + const std::string second_data_file = table_location + "/file2.parquet"; + TDataSink t_data_sink = build_local_delete_sink(temp_dir.string() + "/", 3); + t_data_sink.iceberg_delete_sink.__set_table_location(table_location); VExprContextSPtrs output_exprs; auto sink = std::make_shared(t_data_sink, output_exprs, nullptr, nullptr); ObjectPool pool; @@ -534,23 +646,38 @@ TEST_F(VIcebergDeleteSinkTest, TestWriteDeletionVectorsToSingleSharedPuffin) { std::map file_deletions; auto [file1_it, file1_inserted] = - file_deletions.emplace("file1.parquet", IcebergFileDeletion(1, "[\"p=1\"]")); + file_deletions.emplace(first_data_file, IcebergFileDeletion(1, "[\"p=1\"]")); ASSERT_TRUE(file1_inserted); file1_it->second.rows_to_delete.add((uint32_t)10); file1_it->second.rows_to_delete.add((uint32_t)20); auto [file2_it, file2_inserted] = - file_deletions.emplace("file2.parquet", IcebergFileDeletion(2, "[\"p=2\"]")); + file_deletions.emplace(second_data_file, IcebergFileDeletion(2, "[\"p=2\"]")); ASSERT_TRUE(file2_inserted); file2_it->second.rows_to_delete.add((uint32_t)30); - ASSERT_TRUE(sink->_write_deletion_vector_files(file_deletions).ok()); + const auto status = sink->_write_deletion_vector_files(file_deletions); + ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(2, sink->_commit_data_list.size()); ASSERT_EQ(2, sink->_delete_file_count); + ASSERT_EQ(1, sink->_created_files.size()); + + const std::filesystem::path write_path = sink->_created_files[0].second; + ASSERT_EQ(temp_dir, write_path.parent_path()); + ASSERT_TRUE(std::filesystem::exists(write_path)); const auto& first_commit = sink->_commit_data_list[0]; const auto& second_commit = sink->_commit_data_list[1]; + EXPECT_EQ(table_location + "/" + write_path.filename().string(), first_commit.file_path); ASSERT_EQ(first_commit.file_path, second_commit.file_path); + EXPECT_EQ(first_data_file, first_commit.referenced_data_file_path); + EXPECT_EQ(second_data_file, second_commit.referenced_data_file_path); + EXPECT_EQ(TFileContent::DELETION_VECTOR, first_commit.file_content); + EXPECT_EQ(TFileContent::DELETION_VECTOR, second_commit.file_content); + EXPECT_EQ(1, first_commit.partition_spec_id); + EXPECT_EQ(2, second_commit.partition_spec_id); + EXPECT_EQ("[\"p=1\"]", first_commit.partition_data_json); + EXPECT_EQ("[\"p=2\"]", second_commit.partition_data_json); ASSERT_EQ(first_commit.file_size, second_commit.file_size); ASSERT_LT(first_commit.content_offset, second_commit.content_offset); @@ -562,7 +689,7 @@ TEST_F(VIcebergDeleteSinkTest, TestWriteDeletionVectorsToSingleSharedPuffin) { } ASSERT_EQ(1, puffin_file_count); - std::ifstream input(first_commit.file_path, std::ios::binary); + std::ifstream input(write_path, std::ios::binary); ASSERT_TRUE(input.good()); std::string file_bytes((std::istreambuf_iterator(input)), std::istreambuf_iterator()); diff --git a/be/test/exec/sink/viceberg_merge_sink_test.cpp b/be/test/exec/sink/viceberg_merge_sink_test.cpp index d1da8fb7bfe1d5..27fe8ce745d25e 100644 --- a/be/test/exec/sink/viceberg_merge_sink_test.cpp +++ b/be/test/exec/sink/viceberg_merge_sink_test.cpp @@ -201,6 +201,44 @@ TEST_F(VIcebergMergeSinkTest, TestUpdateProducesDeleteAndInsert) { ASSERT_TRUE(sink->close(Status::OK()).ok()); } +TEST_F(VIcebergMergeSinkTest, TestInnerSinksWriteAndCommitUnderBothFormsOfTheDataLocation) { + ObjectPool pool; + MockRuntimeState state; + + DataTypes types {std::make_shared(), + std::make_shared(DataTypes {std::make_shared(), + std::make_shared()}, + Strings {"file_path", "row_position"}), + std::make_shared(), std::make_shared()}; + MockRowDescriptor row_desc(types, &pool); + auto output_exprs = build_output_exprs(&pool, &state, row_desc); + // UPDATE / MERGE write data files and delete files through one normalized location, and both + // kinds must be committed under the table's own form of it. + TDataSink t_sink = build_sink(); + t_sink.iceberg_merge_sink.__set_output_path("s3://b/db/t/data"); + t_sink.iceberg_merge_sink.__set_original_output_path("oss://b/db/t/data"); + t_sink.iceberg_merge_sink.__set_table_location("oss://b/db/t/data"); + + auto sink = std::make_shared(t_sink, output_exprs, nullptr, nullptr); + sink->set_skip_io(true); + ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok()); + RuntimeProfile profile("iceberg_merge_sink_locations"); + ASSERT_TRUE(sink->open(&state, &profile).ok()); + + ASSERT_TRUE(sink->_table_writer->_output_location.has_value()); + EXPECT_EQ("s3://b/db/t/data/f.parquet", + sink->_table_writer->_output_location->write_path("f.parquet")); + EXPECT_EQ("oss://b/db/t/data/f.parquet", + sink->_table_writer->_output_location->commit_path("f.parquet")); + ASSERT_TRUE(sink->_delete_writer->_output_location.has_value()); + EXPECT_EQ("s3://b/db/t/data/f.parquet", + sink->_delete_writer->_output_location->write_path("f.parquet")); + EXPECT_EQ("oss://b/db/t/data/f.parquet", + sink->_delete_writer->_output_location->commit_path("f.parquet")); + + ASSERT_TRUE(sink->close(Status::OK()).ok()); +} + TEST_F(VIcebergMergeSinkTest, TestDeleteOnlySkipsVariantDataWriter) { ObjectPool pool; MockRuntimeState state; diff --git a/be/test/exec/sink/writer/iceberg/iceberg_output_location_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_output_location_test.cpp new file mode 100644 index 00000000000000..1abd5d71481805 --- /dev/null +++ b/be/test/exec/sink/writer/iceberg/iceberg_output_location_test.cpp @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/iceberg/iceberg_output_location.h" + +#include + +namespace doris { + +TEST(IcebergOutputLocationTest, WritesUnderTheNormalizedFormAndCommitsUnderTheTableForm) { + IcebergOutputLocation location("s3://b/db/t/data", "oss://b/db/t/data"); + + EXPECT_EQ("s3://b/db/t/data/delete_pos_1.parquet", location.write_path("delete_pos_1.parquet")); + EXPECT_EQ("oss://b/db/t/data/delete_pos_1.parquet", + location.commit_path("delete_pos_1.parquet")); +} + +TEST(IcebergOutputLocationTest, JoinsDirectoriesWithAndWithoutTrailingSlashOnce) { + // write.data.path is taken verbatim, so either form may end with '/'. + IcebergOutputLocation location("s3://b/db/t/data/", "oss://b/db/t/data"); + + EXPECT_EQ("s3://b/db/t/data/f.parquet", location.write_path("f.parquet")); + EXPECT_EQ("oss://b/db/t/data/f.parquet", location.commit_path("f.parquet")); + + IcebergOutputLocation partition = location.subdirectory("p=1/q=2"); + EXPECT_EQ("s3://b/db/t/data/p=1/q=2/f.parquet", partition.write_path("f.parquet")); + EXPECT_EQ("oss://b/db/t/data/p=1/q=2/f.parquet", partition.commit_path("f.parquet")); +} + +TEST(IcebergOutputLocationTest, KeepsTheTableFormWhenNormalizationRewritesTheAuthority) { + // FE rewrites more than the scheme for Azure: the account moves out of the authority. + IcebergOutputLocation location( + "s3://container/warehouse/t/data", + "abfss://container@account.dfs.core.windows.net/warehouse/t/data"); + + IcebergOutputLocation partition = location.subdirectory("dt=2024-01-01"); + EXPECT_EQ("s3://container/warehouse/t/data/dt=2024-01-01/f.orc", partition.write_path("f.orc")); + EXPECT_EQ("abfss://container@account.dfs.core.windows.net/warehouse/t/data/dt=2024-01-01/f.orc", + partition.commit_path("f.orc")); +} + +} // namespace doris diff --git a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp index d453177cf25044..5f9e7ea746f013 100644 --- a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp +++ b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp @@ -54,9 +54,12 @@ class VIcebergPartitionWriterTest : public testing::Test { static std::unique_ptr make_writer( const TDataSink& sink, const VExprContextSPtrs& output_exprs, const iceberg::Schema& schema, const std::string* schema_json, - const std::map& hadoop_conf) { - IPartitionWriterBase::WriteInfo write_info; - write_info.file_type = TFileType::FILE_LOCAL; + const std::map& hadoop_conf, + IcebergOutputLocation location = IcebergOutputLocation( + "/tmp/iceberg_partition_writer_test", "/tmp/iceberg_partition_writer_test")) { + IPartitionWriterBase::WriteInfo write_info {.location = std::move(location), + .file_type = TFileType::FILE_LOCAL, + .broker_addresses = {}}; return std::make_unique( sink, std::vector {}, output_exprs, schema, schema_json, std::vector {}, std::move(write_info), "data", 0, @@ -93,6 +96,23 @@ TEST_F(VIcebergPartitionWriterTest, OrcSkipsFooterCollectionWhenMetricsAreDisabl EXPECT_FALSE(commit_data.__isset.column_stats); } +TEST_F(VIcebergPartitionWriterTest, CommitDataRecordsTheTableFormOfTheLocation) { + // The file is written through FE's normalized form; the commit must keep the table's own. + VExprContextSPtrs output_exprs; + iceberg::Schema schema(std::vector {}); + std::string schema_json; + std::map hadoop_conf; + auto writer = + make_writer(make_table_sink(false), output_exprs, schema, &schema_json, hadoop_conf, + IcebergOutputLocation("s3://b/db/t/data/p=1", "oss://b/db/t/data/p=1")); + install_fake_transformer(writer.get(), output_exprs); + + TIcebergCommitData commit_data; + ASSERT_TRUE(build_commit_data(writer.get(), &commit_data).ok()); + EXPECT_EQ("oss://b/db/t/data/p=1/data-0.zlib.orc", commit_data.file_path); + EXPECT_EQ(TFileContent::DATA, commit_data.file_content); +} + TEST_F(VIcebergPartitionWriterTest, MissingPolicyKeepsCollectionEnabledForRollingUpgrade) { VExprContextSPtrs output_exprs; iceberg::Schema schema(std::vector {}); diff --git a/be/test/exec/sink/writer/iceberg/viceberg_table_writer_test.cpp b/be/test/exec/sink/writer/iceberg/viceberg_table_writer_test.cpp index 1bbbe5454af039..29d41501f74905 100644 --- a/be/test/exec/sink/writer/iceberg/viceberg_table_writer_test.cpp +++ b/be/test/exec/sink/writer/iceberg/viceberg_table_writer_test.cpp @@ -32,6 +32,7 @@ #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_varbinary.h" #include "exec/sink/writer/iceberg/partition_transformers.h" +#include "exec/sink/writer/iceberg/viceberg_partition_writer.h" #include "exec/sink/writer/iceberg/vpartition_writer_base.h" #include "exprs/vexpr_context.h" #include "exprs/vslot_ref.h" @@ -260,6 +261,62 @@ TEST(VIcebergTableWriterTest, BinaryIdentityStaticAndDynamicRouting) { EXPECT_EQ(std::vector({"0xdead", "7"}), writer._partition_values(hybrid)); } +TEST(VIcebergTableWriterTest, OpenRejectsASinkWithoutTheTableFormOfTheDataLocation) { + // Without original_output_path the commit used to record relative paths such as + // /p=1/xxx.parquet; now the write fails before any file is created. + TIcebergTableSink sink; + sink.__set_output_path("s3://b/db/t/data"); + TDataSink data_sink; + data_sink.__set_iceberg_table_sink(sink); + VIcebergTableWriter writer(data_sink, {}); + RuntimeState state; + RuntimeProfile profile("test"); + + Status status = writer.open(&state, &profile); + + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(std::string::npos, status.to_string().find("original_output_path")); + EXPECT_FALSE(writer._output_location.has_value()); +} + +TEST(VIcebergTableWriterTest, PartitionWritersWriteAndCommitUnderBothFormsOfTheDataLocation) { + // FE normalizes the location BE writes to (oss:// becomes s3://); the commit must keep the + // table's own form in the data location and in every partition directory under it. + std::vector columns; + columns.emplace_back(false, 1, "p", std::make_unique(), std::nullopt); + TIcebergTableSink sink; + sink.__set_output_path("s3://b/db/t/data"); + sink.__set_original_output_path("oss://b/db/t/data/"); + sink.__set_file_format(TFileFormatType::FORMAT_PARQUET); + sink.__set_compression_type(TFileCompressType::SNAPPYBLOCK); + TDataSink data_sink; + data_sink.__set_iceberg_table_sink(sink); + VIcebergTableWriter writer(data_sink, {}); + writer._schema = std::make_shared(std::move(columns)); + // What open() derives from the two location fields. + writer._output_location.emplace(sink.output_path, sink.original_output_path); + + const std::string file_name = "f"; + auto expect_paths = [&](const std::string& partition_dir) { + auto writer_base = writer._create_partition_writer(nullptr, -1, &file_name, 0); + auto* partition_writer = dynamic_cast(writer_base.get()); + ASSERT_NE(nullptr, partition_writer); + const auto& location = partition_writer->_write_info.location; + const std::string target_file = partition_writer->_get_target_file_name(); + EXPECT_EQ("s3://b/db/t/data/" + partition_dir + target_file, + location.write_path(target_file)); + EXPECT_EQ("oss://b/db/t/data/" + partition_dir + target_file, + location.commit_path(target_file)); + }; + // A non-partitioned table writes into the data location itself. + expect_paths(""); + // A partition writes into its directory under the data location. + writer._is_full_static_partition = true; + writer._static_partition_path = "p=1"; + writer._static_partition_value_list = {"1"}; + expect_paths("p=1/"); +} + TEST(VIcebergTableWriterTest, RejectMissingPartitionSource) { std::vector columns; columns.emplace_back(false, 3, "id", std::make_unique(), std::nullopt); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index 444798b936a978..2d552ac77f6b67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.iceberg.helper.IcebergWriterHelper; import org.apache.doris.nereids.trees.plans.commands.insert.IcebergInsertCommandContext; import org.apache.doris.nereids.trees.plans.commands.insert.InsertCommandContext; +import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TFileContent; import org.apache.doris.thrift.TIcebergCommitData; import org.apache.doris.thrift.TUpdateMode; @@ -78,6 +79,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.BooleanSupplier; public class IcebergTransaction implements Transaction { @@ -92,6 +94,7 @@ public class IcebergTransaction implements Transaction { private org.apache.iceberg.Transaction transaction; private final List commitDataList = Lists.newArrayList(); + private final Set commitLocations = new HashSet<>(); private Optional conflictDetectionFilter = Optional.empty(); private IcebergInsertCommandContext insertCtx; @@ -117,6 +120,42 @@ public void updateIcebergCommitData(List commitDataList) { } } + /** Register the raw locations actually sent to BE, including every group of a rewrite. */ + public synchronized void addCommitLocations(TDataSink sink) { + switch (sink.getType()) { + case ICEBERG_TABLE_SINK: + addCommitLocation(sink.getIcebergTableSink().getOriginalOutputPath()); + break; + case ICEBERG_DELETE_SINK: + addCommitLocation(sink.getIcebergDeleteSink().getTableLocation()); + break; + case ICEBERG_MERGE_SINK: + addCommitLocation(sink.getIcebergMergeSink().getOriginalOutputPath()); + addCommitLocation(sink.getIcebergMergeSink().getTableLocation()); + break; + default: + throw new IllegalArgumentException("Not an Iceberg sink: " + sink.getType()); + } + } + + private void addCommitLocation(String location) { + Preconditions.checkArgument(location != null && !location.isEmpty(), + "Iceberg sink is missing its raw commit location"); + commitLocations.add(location.endsWith("/") ? location : location + "/"); + } + + // These branches do not acknowledge file reports. Validate after collection, before staging + // Iceberg updates, so rejection fails the statement through its normal transaction error path. + private synchronized void validateCommitLocations() { + for (TIcebergCommitData data : commitDataList) { + String path = data.getFilePath(); + Preconditions.checkState(path != null && !path.isEmpty(), + "Iceberg commit data is missing its file path"); + Preconditions.checkState(commitLocations.stream().anyMatch(path::startsWith), + "Iceberg commit file %s is outside the sink locations %s", path, commitLocations); + } + } + public void setConflictDetectionFilter(Expression filter) { conflictDetectionFilter = Optional.ofNullable(filter); } @@ -244,6 +283,7 @@ public void beginRewrite(ExternalTable dorisTable, Table targetTable, */ public void finishRewrite() { try { + validateCommitLocations(); writableAuthenticator().execute(() -> { convertCommitDataListToDataFilesToAdd(); if (LOG.isDebugEnabled()) { @@ -451,6 +491,7 @@ public void finishDelete(NameMapping nameMapping) { LOG.debug("iceberg table {} delete operation finished!", nameMapping.getFullLocalName()); } try { + validateCommitLocations(); writableAuthenticator().execute(() -> { updateManifestAfterDelete(); }); @@ -468,6 +509,7 @@ public void finishMerge(NameMapping nameMapping) { LOG.debug("iceberg table {} merge operation finished!", nameMapping.getFullLocalName()); } try { + validateCommitLocations(); writableAuthenticator().execute(() -> { updateManifestAfterMerge(); }); @@ -628,6 +670,7 @@ public void finishInsert(NameMapping nameMapping) { LOG.info("iceberg table {} insert table finished!", nameMapping.getFullLocalName()); } try { + validateCommitLocations(); writableAuthenticator().execute(() -> { //create and start the iceberg transaction TUpdateMode updateMode = TUpdateMode.APPEND; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java index 29b05bfb1cec80..60620a962eeea3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java @@ -221,8 +221,8 @@ private void executeGroup(ConnectContext taskConnectContext, Preconditions.checkState(insertExecutor instanceof IcebergRewriteExecutor, "Expected IcebergRewriteExecutor, got: " + insertExecutor.getClass()); - // Step 3: Set transaction id for updating CommitData - insertExecutor.getCoordinator().setTxnId(transactionId); + // Step 3: Register this group's sink location with the shared transaction before execution. + ((IcebergRewriteExecutor) insertExecutor).bindTransaction(transactionId); // Step 4: Publish coordinator so cancel() can reach the running BE tasks stmtExecutor.setCoord(insertExecutor.getCoordinator()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java index 288bcb32f5fc46..3f78cf2cafa95f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java @@ -53,6 +53,7 @@ public abstract class BaseExternalTableInsertExecutor extends AbstractInsertExec private static final Logger LOG = LogManager.getLogger(BaseExternalTableInsertExecutor.class); protected TransactionStatus txnStatus = TransactionStatus.ABORTED; protected final TransactionManager transactionManager; + protected BaseExternalTableDataSink dataSink; protected final String catalogName; protected Optional summaryProfile = Optional.empty(); @@ -144,7 +145,8 @@ protected void doAfterCommit() throws DdlException { @Override protected void finalizeSink(PlanFragment fragment, DataSink sink, PhysicalSink physicalSink) { try { - ((BaseExternalTableDataSink) sink).bindDataSink(insertCtx); + dataSink = (BaseExternalTableDataSink) sink; + dataSink.bindDataSink(insertCtx); } catch (Exception e) { throw new AnalysisException(e.getMessage(), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutor.java index 7b6863e9ba58c5..4816222c2395b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutor.java @@ -91,6 +91,7 @@ protected void beforeExec() throws UserException { IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId); // DELETE conflict validation must start from the same retained generation scanned by BE. transaction.beginDelete((IcebergExternalTable) table, targetIcebergTable); + transaction.addCommitLocations(dataSink.toThrift()); transaction.setRewrittenDeleteFilesByReferencedDataFile( rewritableDeletePlan.getDeleteFilesByReferencedDataFile()); if (conflictDetectionFilter.isPresent()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java index 59b0fdd8e4f33a..4e2423fb42a926 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java @@ -55,6 +55,10 @@ public IcebergInsertExecutor(ConnectContext ctx, IcebergExternalTable table, protected void beforeExec() throws UserException { IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId); transaction.beginInsert((IcebergExternalTable) table, targetIcebergTable, insertCtx); + // Empty static overwrites execute without a BE sink or newly written files. + if (!emptyInsert) { + transaction.addCommitLocations(dataSink.toThrift()); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutor.java index 338d58dd10dc06..31a301f8fef327 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutor.java @@ -93,6 +93,7 @@ public void setConflictDetectionFilter(Optional filter) { protected void beforeExec() throws UserException { IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId); transaction.beginMerge((IcebergExternalTable) table, targetIcebergTable, insertCtx); + transaction.addCommitLocations(dataSink.toThrift()); transaction.setRewrittenDeleteFilesByReferencedDataFile( rewritableDeletePlan.getDeleteFilesByReferencedDataFile()); if (conflictDetectionFilter.isPresent()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergRewriteExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergRewriteExecutor.java index fe0efc98b8f8fa..98b8db3a5f8224 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergRewriteExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergRewriteExecutor.java @@ -19,6 +19,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergTransaction; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.qe.ConnectContext; import org.apache.doris.transaction.TransactionType; @@ -43,6 +44,13 @@ public IcebergRewriteExecutor(ConnectContext ctx, IcebergExternalTable table, super(ctx, table, labelName, planner, insertCtx, emptyInsert, jobId); } + /** Bind the enclosing rewrite transaction after this group's sink has been finalized. */ + public void bindTransaction(long transactionId) throws UserException { + IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(transactionId); + transaction.addCommitLocations(dataSink.toThrift()); + coordinator.setTxnId(transactionId); + } + @Override protected void beforeExec() throws UserException { // do nothing, the transaction is not managed by IcebergRewriteExecutor diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/BaseExternalTableDataSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/BaseExternalTableDataSink.java index 433162f93fe94c..d14111e97c0fe8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/BaseExternalTableDataSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/BaseExternalTableDataSink.java @@ -40,7 +40,7 @@ public abstract class BaseExternalTableDataSink extends DataSink { protected TDataSink tDataSink; @Override - protected TDataSink toThrift() { + public TDataSink toThrift() { return tDataSink; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergCommitLocationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergCommitLocationTest.java new file mode 100644 index 00000000000000..4633c755515f0b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergCommitLocationTest.java @@ -0,0 +1,260 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.nereids.trees.plans.commands.insert.IcebergInsertCommandContext; +import org.apache.doris.thrift.TDataSink; +import org.apache.doris.thrift.TDataSinkType; +import org.apache.doris.thrift.TFileContent; +import org.apache.doris.thrift.TIcebergCommitData; +import org.apache.doris.thrift.TIcebergDeleteSink; +import org.apache.doris.thrift.TIcebergMergeSink; +import org.apache.doris.thrift.TIcebergTableSink; + +import org.apache.iceberg.AppendFiles; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.ReplacePartitions; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; + +class IcebergCommitLocationTest { + private static final String LOCATION = "abfss://container@account.dfs.core.windows.net/custom/data"; + private static final String NORMALIZED_LOCATION = "s3://container/custom/data"; + + enum WriteKind { + INSERT, OVERWRITE, DELETE, MERGE, REWRITE + } + + @ParameterizedTest + @EnumSource(WriteKind.class) + void testAcceptRawLocationWithAndWithoutTrailingSlash(WriteKind kind) throws Exception { + for (String location : Arrays.asList(LOCATION, LOCATION + "/")) { + try (Fixture fixture = new Fixture(kind)) { + fixture.txn.addCommitLocations(sink(kind, location, location)); + fixture.txn.updateIcebergCommitData(Collections.singletonList( + file(LOCATION + "/p=1/file.parquet", kind))); + fixture.finish(); + fixture.txn.commit(); + Mockito.verify(fixture.nativeTxn).commitTransaction(); + } + } + } + + @ParameterizedTest + @EnumSource(WriteKind.class) + void testRejectEntireBatchBeforeStagingAnyFiles(WriteKind kind) throws Exception { + for (String path : Arrays.asList(NORMALIZED_LOCATION + "/file.parquet", + LOCATION + "2/file.parquet", LOCATION, "file.parquet")) { + try (Fixture fixture = new Fixture(kind)) { + fixture.txn.addCommitLocations(sink(kind, LOCATION, LOCATION)); + fixture.txn.updateIcebergCommitData(Arrays.asList( + file(LOCATION + "/valid.parquet", kind), file(path, kind))); + RuntimeException error = Assertions.assertThrows(RuntimeException.class, fixture::finish); + Assertions.assertTrue(error.getMessage().contains("outside the sink locations")); + Assertions.assertTrue(error.getMessage().contains(path)); + Mockito.verifyNoInteractions(fixture.nativeTxn); + } + } + } + + @ParameterizedTest + @EnumSource(WriteKind.class) + void testRejectMissingFilePath(WriteKind kind) throws Exception { + for (String path : Arrays.asList(null, "")) { + try (Fixture fixture = new Fixture(kind)) { + fixture.txn.addCommitLocations(sink(kind, LOCATION, LOCATION)); + fixture.txn.updateIcebergCommitData(Collections.singletonList(file(path, kind))); + RuntimeException error = Assertions.assertThrows(RuntimeException.class, fixture::finish); + Assertions.assertTrue(error.getMessage().contains("missing its file path")); + Mockito.verifyNoInteractions(fixture.nativeTxn); + } + } + } + + @ParameterizedTest + @EnumSource(WriteKind.class) + void testRejectFilesWhenSinkWasNotRegistered(WriteKind kind) throws Exception { + try (Fixture fixture = new Fixture(kind)) { + fixture.txn.updateIcebergCommitData(Collections.singletonList(file(LOCATION + "/file.parquet", kind))); + RuntimeException error = Assertions.assertThrows(RuntimeException.class, fixture::finish); + Assertions.assertTrue(error.getMessage().contains("outside the sink locations")); + Mockito.verifyNoInteractions(fixture.nativeTxn); + } + } + + @ParameterizedTest + @EnumSource(WriteKind.class) + void testRejectMissingRawSinkLocation(WriteKind kind) { + IcebergTransaction txn = new IcebergTransaction(Mockito.mock(IcebergMetadataOps.class)); + for (String location : Arrays.asList(null, "")) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> txn.addCommitLocations(sink(kind, location, location))); + } + } + + @Test + void testMergeRegistersBothRawLocations() throws Exception { + try (Fixture fixture = new Fixture(WriteKind.MERGE)) { + fixture.txn.addCommitLocations(sink(WriteKind.MERGE, LOCATION + "/insert", LOCATION + "/delete/")); + fixture.txn.updateIcebergCommitData(Arrays.asList( + file(LOCATION + "/insert/data.parquet", WriteKind.INSERT), + file(LOCATION + "/delete/pos.parquet", WriteKind.DELETE))); + fixture.finish(); + fixture.txn.commit(); + Mockito.verify(fixture.nativeTxn).commitTransaction(); + } + } + + @Test + void testRewriteAccumulatesLocationsFromAllGroups() throws Exception { + try (Fixture fixture = new Fixture(WriteKind.REWRITE)) { + fixture.txn.addCommitLocations(sink(WriteKind.REWRITE, LOCATION + "/first", LOCATION)); + fixture.txn.addCommitLocations(sink(WriteKind.REWRITE, LOCATION + "/second", LOCATION)); + fixture.txn.updateIcebergCommitData(Arrays.asList( + file(LOCATION + "/first/data.parquet", WriteKind.REWRITE), + file(LOCATION + "/second/data.parquet", WriteKind.REWRITE))); + fixture.finish(); + fixture.txn.commit(); + Mockito.verify(fixture.nativeTxn).commitTransaction(); + } + } + + private static TDataSink sink(WriteKind kind, String dataLocation, String deleteLocation) { + switch (kind) { + case DELETE: + return new TDataSink(TDataSinkType.ICEBERG_DELETE_SINK).setIcebergDeleteSink( + new TIcebergDeleteSink().setTableLocation(deleteLocation) + .setOutputPath(NORMALIZED_LOCATION)); + case MERGE: + return new TDataSink(TDataSinkType.ICEBERG_MERGE_SINK).setIcebergMergeSink( + new TIcebergMergeSink().setOriginalOutputPath(dataLocation).setTableLocation(deleteLocation) + .setOutputPath(NORMALIZED_LOCATION)); + default: + return new TDataSink(TDataSinkType.ICEBERG_TABLE_SINK).setIcebergTableSink( + new TIcebergTableSink().setOriginalOutputPath(dataLocation) + .setOutputPath(NORMALIZED_LOCATION)); + } + } + + private static TIcebergCommitData file(String path, WriteKind kind) { + return new TIcebergCommitData().setFilePath(path).setRowCount(1).setFileSize(128) + .setFileContent(kind == WriteKind.DELETE ? TFileContent.POSITION_DELETES : TFileContent.DATA); + } + + private static class Fixture implements AutoCloseable { + private final WriteKind kind; + private final IcebergTransaction txn; + private final org.apache.iceberg.Transaction nativeTxn = Mockito.mock(org.apache.iceberg.Transaction.class); + private final MockedStatic utils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS); + + Fixture(WriteKind kind) throws Exception { + this.kind = kind; + Table table = Mockito.mock(Table.class); + PartitionSpec spec = PartitionSpec.unpartitioned(); + Mockito.when(table.schema()).thenReturn(new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()))); + Mockito.when(table.spec()).thenReturn(spec); + Mockito.when(table.specs()).thenReturn(Collections.singletonMap(spec.specId(), spec)); + Mockito.when(table.properties()).thenReturn(Collections.singletonMap(TableProperties.FORMAT_VERSION, "2")); + Mockito.when(table.name()).thenReturn("db.table"); + Mockito.when(table.newTransaction()).thenReturn(nativeTxn); + Mockito.when(nativeTxn.table()).thenReturn(table); + AppendFiles append = Mockito.mock(AppendFiles.class, Mockito.RETURNS_SELF); + Mockito.when(append.scanManifestsWith(Mockito.any())).thenReturn(append); + Mockito.when(nativeTxn.newAppend()).thenReturn(append); + ReplacePartitions overwrite = Mockito.mock(ReplacePartitions.class, Mockito.RETURNS_SELF); + Mockito.when(overwrite.scanManifestsWith(Mockito.any())).thenReturn(overwrite); + Mockito.when(nativeTxn.newReplacePartitions()).thenReturn(overwrite); + Mockito.when(nativeTxn.newRowDelta()).thenReturn(Mockito.mock(RowDelta.class, Mockito.RETURNS_SELF)); + RewriteFiles rewrite = Mockito.mock(RewriteFiles.class, Mockito.RETURNS_SELF); + Mockito.when(rewrite.scanManifestsWith(Mockito.any())).thenReturn(rewrite); + Mockito.when(nativeTxn.newRewrite()).thenReturn(rewrite); + + IcebergMetadataOps ops = Mockito.mock(IcebergMetadataOps.class); + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getName()).thenReturn("table"); + IcebergRuntimeContext runtime = Mockito.mock(IcebergRuntimeContext.class); + Mockito.when(runtime.getAuthenticator()).thenReturn(new ExecutionAuthenticator() { }); + utils.when(() -> IcebergUtils.acquireWritableIcebergTable(dorisTable, ops)) + .thenAnswer(invocation -> new IcebergExternalMetaCache.WritableTableLease( + table, ops, runtime, false, false, () -> { })); + txn = new IcebergTransaction(ops); + switch (kind) { + case DELETE: + txn.beginDelete(dorisTable, table); + break; + case MERGE: + txn.beginMerge(dorisTable, table, Optional.empty()); + break; + case REWRITE: + txn.beginRewrite(dorisTable, table); + break; + default: + IcebergInsertCommandContext context = new IcebergInsertCommandContext(); + context.setOverwrite(kind == WriteKind.OVERWRITE); + txn.beginInsert(dorisTable, table, Optional.of(context)); + break; + } + Mockito.clearInvocations(nativeTxn); + } + + void finish() { + NameMapping name = NameMapping.createForTest("db", "table"); + switch (kind) { + case DELETE: + txn.finishDelete(name); + break; + case MERGE: + txn.finishMerge(name); + break; + case REWRITE: + txn.finishRewrite(); + break; + default: + txn.finishInsert(name); + break; + } + } + + @Override + public void close() { + try { + txn.rollback(); + } finally { + utils.close(); + } + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index e9300c82d769c8..ed0bdf9b948c5c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -26,8 +26,11 @@ import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.foundation.util.SerializationUtils; import org.apache.doris.nereids.trees.plans.commands.insert.IcebergInsertCommandContext; +import org.apache.doris.thrift.TDataSink; +import org.apache.doris.thrift.TDataSinkType; import org.apache.doris.thrift.TFileContent; import org.apache.doris.thrift.TIcebergCommitData; +import org.apache.doris.thrift.TIcebergTableSink; import com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; @@ -205,18 +208,19 @@ private List createPartitionValues(Instant instant, String str1, String @Test public void testPartitionedTable() throws UserException { + Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithPartition)); List partitionValues = createPartitionValues(); List ctdList = new ArrayList<>(); TIcebergCommitData ctd1 = new TIcebergCommitData(); - ctd1.setFilePath("f1.parquet"); + ctd1.setFilePath(table.location() + "/data/f1.parquet"); ctd1.setPartitionValues(partitionValues); ctd1.setFileContent(TFileContent.DATA); ctd1.setRowCount(2); ctd1.setFileSize(2); TIcebergCommitData ctd2 = new TIcebergCommitData(); - ctd2.setFilePath("f2.parquet"); + ctd2.setFilePath(table.location() + "/data/f2.parquet"); ctd2.setPartitionValues(partitionValues); ctd2.setFileContent(TFileContent.DATA); ctd2.setRowCount(4); @@ -225,8 +229,6 @@ public void testPartitionedTable() throws UserException { ctdList.add(ctd1); ctdList.add(ctd2); - Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithPartition)); - IcebergExternalTable icebergExternalTable = Mockito.mock(IcebergExternalTable.class); Mockito.when(icebergExternalTable.getCatalog()).thenReturn(spyExternalCatalog); Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); @@ -238,7 +240,7 @@ public void testPartitionedTable() throws UserException { mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) .thenCallRealMethod(); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(table.location() + "/data"); txn.updateIcebergCommitData(ctdList); txn.beginInsert(icebergExternalTable, table, Optional.empty()); txn.finishInsert(NameMapping.createForTest(dbName, tbWithPartition)); @@ -325,15 +327,16 @@ private void checkPushDownByPartition(Table table, Expression expr, Integer expe @Test public void testUnPartitionedTable() throws UserException { + Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); ArrayList ctdList = new ArrayList<>(); TIcebergCommitData ctd1 = new TIcebergCommitData(); - ctd1.setFilePath("f1.parquet"); + ctd1.setFilePath(table.location() + "/data/f1.parquet"); ctd1.setFileContent(TFileContent.DATA); ctd1.setRowCount(2); ctd1.setFileSize(2); TIcebergCommitData ctd2 = new TIcebergCommitData(); - ctd2.setFilePath("f2.parquet"); + ctd2.setFilePath(table.location() + "/data/f2.parquet"); ctd2.setFileContent(TFileContent.DATA); ctd2.setRowCount(4); ctd2.setFileSize(4); @@ -341,7 +344,6 @@ public void testUnPartitionedTable() throws UserException { ctdList.add(ctd1); ctdList.add(ctd2); - Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); IcebergExternalTable icebergExternalTable = Mockito.mock(IcebergExternalTable.class); Mockito.when(icebergExternalTable.getCatalog()).thenReturn(spyExternalCatalog); Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); @@ -350,7 +352,7 @@ public void testUnPartitionedTable() throws UserException { try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { mockWritableTable(mockedStatic, table); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(table.location() + "/data"); txn.updateIcebergCommitData(ctdList); txn.beginInsert(icebergExternalTable, table, Optional.empty()); txn.finishInsert(NameMapping.createForTest(dbName, tbWithPartition)); @@ -364,6 +366,14 @@ private IcebergTransaction getTxn() { return new IcebergTransaction(ops); } + private IcebergTransaction getTxn(String dataLocation) { + IcebergTransaction txn = getTxn(); + TIcebergTableSink tableSink = new TIcebergTableSink(); + tableSink.setOriginalOutputPath(dataLocation); + txn.addCommitLocations(new TDataSink(TDataSinkType.ICEBERG_TABLE_SINK).setIcebergTableSink(tableSink)); + return txn; + } + private void mockWritableTable(MockedStatic mockedUtils, Table table) { mockedUtils.when(() -> IcebergUtils.acquireWritableIcebergTable( ArgumentMatchers.any(ExternalTable.class), ArgumentMatchers.eq(ops))) @@ -561,7 +571,7 @@ private void verifyDynamicOverwriteRejectsPartitionedToUnpartitionedSpecDrift( insertContext.setOverwrite(true); insertContext.setWriteSchemaContext(Optional.of(context)); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(table.location() + "/data"); if (hasOutputFile) { TIcebergCommitData commitData = new TIcebergCommitData(); commitData.setFilePath(table.location() + "/data/output.parquet"); @@ -619,7 +629,7 @@ public void testCommitReplayRejectsRequiredSchemaChangeAfterStaging() throws Use commitData.setRowCount(1); commitData.setFileSize(1); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(table.location() + "/data"); txn.updateIcebergCommitData(Collections.singletonList(commitData)); try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { @@ -663,7 +673,7 @@ public void testMergeCommitReplayRejectsRequiredSchemaChangeAfterStaging() throw commitData.setRowCount(1); commitData.setFileSize(1); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(table.location() + "/data"); try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { mockWritableTable(mockedUtils, table); @@ -704,7 +714,7 @@ public void testCommitRejectsTableReplacementAfterStaging() throws UserException commitData.setRowCount(1); commitData.setFileSize(1); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(table.location() + "/data"); txn.updateIcebergCommitData(Collections.singletonList(commitData)); try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { @@ -764,7 +774,7 @@ public void testInsertCommitUsesStatementPinnedWriterMetadata() throws UserExcep IcebergInsertCommandContext insertContext = new IcebergInsertCommandContext(); insertContext.setWriteSchemaContext(Optional.of(context)); TIcebergCommitData commitData = new TIcebergCommitData(); - commitData.setFilePath("file:///tmp/pinned/data.parquet"); + commitData.setFilePath(dataLocation + "/data.parquet"); commitData.setRowCount(1); commitData.setFileSize(128); WriteResult writeResult = WriteResult.builder().addDataFiles(dataFile).build(); @@ -779,7 +789,7 @@ public void testInsertCommitUsesStatementPinnedWriterMetadata() throws UserExcep ArgumentMatchers.same(context), ArgumentMatchers.anyList())) .thenReturn(writeResult); - IcebergTransaction txn = Mockito.spy(getTxn()); + IcebergTransaction txn = Mockito.spy(getTxn(dataLocation)); Mockito.doReturn(icebergTxn).when(txn).newWriteTransaction(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, Optional.of(insertContext)); @@ -856,24 +866,25 @@ public void testTransform() { @Test public void testUnPartitionedTableOverwriteWithData() throws UserException { + Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); testUnPartitionedTable(); ArrayList ctdList = new ArrayList<>(); TIcebergCommitData ctd1 = new TIcebergCommitData(); - ctd1.setFilePath("f3.parquet"); + ctd1.setFilePath(table.location() + "/data/f3.parquet"); ctd1.setFileContent(TFileContent.DATA); ctd1.setRowCount(6); ctd1.setFileSize(6); TIcebergCommitData ctd2 = new TIcebergCommitData(); - ctd2.setFilePath("f4.parquet"); + ctd2.setFilePath(table.location() + "/data/f4.parquet"); ctd2.setFileContent(TFileContent.DATA); ctd2.setRowCount(8); ctd2.setFileSize(8); TIcebergCommitData ctd3 = new TIcebergCommitData(); - ctd3.setFilePath("f5.parquet"); + ctd3.setFilePath(table.location() + "/data/f5.parquet"); ctd3.setFileContent(TFileContent.DATA); ctd3.setRowCount(10); ctd3.setFileSize(10); @@ -882,7 +893,6 @@ public void testUnPartitionedTableOverwriteWithData() throws UserException { ctdList.add(ctd2); ctdList.add(ctd3); - Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); IcebergExternalTable icebergExternalTable = Mockito.mock(IcebergExternalTable.class); Mockito.when(icebergExternalTable.getCatalog()).thenReturn(spyExternalCatalog); Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); @@ -890,7 +900,7 @@ public void testUnPartitionedTableOverwriteWithData() throws UserException { try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { mockWritableTable(mockedStatic, table); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(table.location() + "/data"); txn.updateIcebergCommitData(ctdList); IcebergInsertCommandContext ctx = new IcebergInsertCommandContext(); txn.beginInsert(icebergExternalTable, table, Optional.of(ctx)); @@ -915,7 +925,7 @@ public void testUnpartitionedTableOverwriteWithoutData() throws UserException { try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { mockWritableTable(mockedStatic, table); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(table.location() + "/data"); IcebergInsertCommandContext ctx = new IcebergInsertCommandContext(); txn.beginInsert(icebergExternalTable, table, Optional.of(ctx)); ctx.setOverwrite(true); @@ -928,9 +938,10 @@ public void testUnpartitionedTableOverwriteWithoutData() throws UserException { @Test public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() throws UserException { + Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithPartition)); List ctdList = new ArrayList<>(); TIcebergCommitData ctd1 = new TIcebergCommitData(); - ctd1.setFilePath("partition-a.parquet"); + ctd1.setFilePath(table.location() + "/data/partition-a.parquet"); ctd1.setPartitionValues(createPartitionValues( Instant.parse("2024-12-11T12:34:56.123456Z"), "partition-a", "truncate-a", 11)); @@ -939,7 +950,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th ctd1.setFileSize(2); TIcebergCommitData ctd2 = new TIcebergCommitData(); - ctd2.setFilePath("partition-b.parquet"); + ctd2.setFilePath(table.location() + "/data/partition-b.parquet"); ctd2.setPartitionValues(createPartitionValues( Instant.parse("2024-12-12T12:34:56.123456Z"), "partition-b", "truncate-b", 25)); @@ -950,7 +961,6 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th ctdList.add(ctd1); ctdList.add(ctd2); - Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithPartition)); IcebergWriteSchemaContext writeSchemaContext = IcebergWriteSchemaContext.forSchema( table.schema(), IcebergUtils.getFormatVersion(table), table.spec(), table.sortOrder(), IcebergUtils.getFileFormat(table), MetricsConfig.forTable(table), @@ -966,7 +976,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) .thenCallRealMethod(); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(table.location() + "/data"); txn.updateIcebergCommitData(ctdList); txn.beginInsert(icebergExternalTable, table, Optional.empty()); txn.finishInsert(NameMapping.createForTest(dbName, tbWithPartition)); @@ -988,7 +998,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th ArgumentMatchers.any(Table.class))) .thenReturn(writeSchemaContext.getDataLocation()); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(table.location() + "/data"); IcebergInsertCommandContext ctx = new IcebergInsertCommandContext(); ctx.setOverwrite(true); Map staticPartitions = new LinkedHashMap<>(); @@ -1042,7 +1052,7 @@ public void testFinishDeleteRewritesAllSharedPuffinDeleteFilesForV3() throws Use Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); TIcebergCommitData commitData = new TIcebergCommitData(); - commitData.setFilePath("delete-dv-shared.puffin"); + commitData.setFilePath("s3a://warehouse/wh/db3/tbWithoutPartition/data/delete-dv-shared.puffin"); commitData.setFileContent(TFileContent.POSITION_DELETES); commitData.setRowCount(3); commitData.setFileSize(44); @@ -1050,7 +1060,7 @@ public void testFinishDeleteRewritesAllSharedPuffinDeleteFilesForV3() throws Use commitData.setContentSizeInBytes(21); commitData.setReferencedDataFilePath(referencedDataFile); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn("s3a://warehouse/wh/db3/tbWithoutPartition/data"); txn.updateIcebergCommitData(Collections.singletonList(commitData)); try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); @@ -1104,13 +1114,13 @@ private void verifyFinishDeleteRewriteBehavior(int formatVersion, boolean expect Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); TIcebergCommitData commitData = new TIcebergCommitData(); - commitData.setFilePath("delete-dv.puffin"); + commitData.setFilePath("s3a://warehouse/wh/db3/tbWithoutPartition/data/delete-dv.puffin"); commitData.setFileContent(TFileContent.POSITION_DELETES); commitData.setRowCount(3); commitData.setFileSize(33); commitData.setReferencedDataFilePath(referencedDataFile); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn("s3a://warehouse/wh/db3/tbWithoutPartition/data"); txn.updateIcebergCommitData(Collections.singletonList(commitData)); try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); @@ -1257,7 +1267,7 @@ public void testQueryScopedGenerationCommitsThroughWritableOperations() throws U Mockito.when(dorisTable.getName()).thenReturn(tbWithoutPartition); TIcebergCommitData commitData = new TIcebergCommitData(); - commitData.setFilePath("query-scoped-generation.parquet"); + commitData.setFilePath(liveTable.location() + "/data/query-scoped-generation.parquet"); commitData.setFileContent(TFileContent.DATA); commitData.setRowCount(1); commitData.setFileSize(1); @@ -1265,7 +1275,7 @@ public void testQueryScopedGenerationCommitsThroughWritableOperations() throws U try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { mockWritableTable(mockedUtils, liveTable); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(liveTable.location() + "/data"); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, queryScopedTable, Optional.empty()); txn.finishInsert(NameMapping.createForTest(dbName, tbWithoutPartition)); @@ -1287,7 +1297,7 @@ public void testRetainedGenerationCommitsThroughWritableOperations() throws User Mockito.when(dorisTable.getName()).thenReturn(tbWithoutPartition); TIcebergCommitData commitData = new TIcebergCommitData(); - commitData.setFilePath("retained-generation.parquet"); + commitData.setFilePath(liveTable.location() + "/data/retained-generation.parquet"); commitData.setFileContent(TFileContent.DATA); commitData.setRowCount(1); commitData.setFileSize(1); @@ -1295,7 +1305,7 @@ public void testRetainedGenerationCommitsThroughWritableOperations() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { mockWritableTable(mockedUtils, liveTable); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(liveTable.location() + "/data"); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); txn.finishInsert(NameMapping.createForTest(dbName, tbWithoutPartition)); @@ -1337,7 +1347,7 @@ public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws User Mockito.when(dorisTable.getName()).thenReturn(tbWithoutPartition); TIcebergCommitData commitData = new TIcebergCommitData(); - commitData.setFilePath("stale-retained-generation.parquet"); + commitData.setFilePath(liveTable.location() + "/data/stale-retained-generation.parquet"); commitData.setFileContent(TFileContent.DATA); commitData.setRowCount(1); commitData.setFileSize(1); @@ -1345,7 +1355,7 @@ public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { mockWritableTable(mockedUtils, liveTable); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(liveTable.location() + "/data"); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); txn.finishInsert(NameMapping.createForTest(dbName, tbWithoutPartition)); @@ -1366,7 +1376,7 @@ public void testRetainedGenerationRetriesAfterConcurrentDataCommit() throws User Mockito.when(dorisTable.getName()).thenReturn(tbWithoutPartition); TIcebergCommitData commitData = new TIcebergCommitData(); - commitData.setFilePath("retry-after-concurrent-data-commit.parquet"); + commitData.setFilePath(liveTable.location() + "/data/retry-after-concurrent-data-commit.parquet"); commitData.setFileContent(TFileContent.DATA); commitData.setRowCount(1); commitData.setFileSize(1); @@ -1374,7 +1384,7 @@ public void testRetainedGenerationRetriesAfterConcurrentDataCommit() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { mockWritableTable(mockedUtils, liveTable); - IcebergTransaction txn = getTxn(); + IcebergTransaction txn = getTxn(liveTable.location() + "/data"); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); txn.finishInsert(NameMapping.createForTest(dbName, tbWithoutPartition)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergCommitLocationRegistrationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergCommitLocationRegistrationTest.java new file mode 100644 index 00000000000000..6bd63c983ee773 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergCommitLocationRegistrationTest.java @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.datasource.CatalogProperty; +import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergTransaction; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.planner.IcebergTableSink; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.Coordinator; +import org.apache.doris.thrift.TUniqueId; +import org.apache.doris.transaction.TransactionManager; + +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Optional; + +class IcebergCommitLocationRegistrationTest { + @Test + void testInsertRegistersTheBoundSinkBeforeExecution() throws Exception { + try (Fixture fixture = new Fixture()) { + IcebergInsertExecutor executor = new IcebergInsertExecutor(fixture.context, fixture.table, + fixture.nativeTable, "label", Mockito.mock(NereidsPlanner.class), Optional.empty(), false, -1L); + executor.txnId = 101L; + IcebergTableSink sink = new IcebergTableSink(fixture.table, fixture.nativeTable); + executor.finalizeSink(null, sink, null); + + executor.beforeExec(); + + Mockito.verify(fixture.transaction).beginInsert(fixture.table, fixture.nativeTable, Optional.empty()); + Mockito.verify(fixture.transaction).addCommitLocations(sink.toThrift()); + Assertions.assertEquals("file:///tmp/custom/data/", + sink.toThrift().getIcebergTableSink().getOriginalOutputPath()); + } + } + + @Test + void testEmptyStaticOverwriteDoesNotRequireABeSink() throws Exception { + try (Fixture fixture = new Fixture()) { + IcebergInsertCommandContext context = new IcebergInsertCommandContext(); + context.setOverwrite(true); + context.setStaticPartitionValues(Collections.singletonMap("partition_key", "1")); + IcebergInsertExecutor executor = new IcebergInsertExecutor(fixture.context, fixture.table, + fixture.nativeTable, "label", Mockito.mock(NereidsPlanner.class), Optional.of(context), true, -1L); + executor.txnId = 101L; + Assertions.assertTrue(executor.shouldExecuteEmptyInsert()); + + executor.beforeExec(); + + Mockito.verify(fixture.transaction).beginInsert( + fixture.table, fixture.nativeTable, Optional.of(context)); + Mockito.verify(fixture.transaction, Mockito.never()).addCommitLocations(Mockito.any()); + } + } + + @Test + void testRewriteRegistersAfterBindingTheSharedTransaction() throws Exception { + try (Fixture fixture = new Fixture()) { + IcebergInsertCommandContext context = new IcebergInsertCommandContext(); + context.setRewriting(true); + IcebergRewriteExecutor executor = new IcebergRewriteExecutor(fixture.context, fixture.table, + "label", Mockito.mock(NereidsPlanner.class), Optional.of(context), false, -1L); + IcebergTableSink sink = new IcebergTableSink(fixture.table, fixture.nativeTable); + executor.finalizeSink(null, sink, null); + Mockito.verifyNoInteractions(fixture.transactionManager); + + executor.bindTransaction(101L); + + Mockito.verify(fixture.transaction).addCommitLocations(sink.toThrift()); + Mockito.verify(fixture.coordinator).setTxnId(101L); + // A group must not take over committing the enclosing rewrite transaction. + Assertions.assertEquals(-1L, executor.getTxnId()); + } + } + + private static class Fixture implements AutoCloseable { + private final ConnectContext context = new ConnectContext(); + private final IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + private final Table nativeTable = Mockito.mock(Table.class); + private final IcebergTransaction transaction = Mockito.mock(IcebergTransaction.class); + private final TransactionManager transactionManager = Mockito.mock(TransactionManager.class); + private final Coordinator coordinator = Mockito.mock(Coordinator.class); + private final MockedStatic factoryMock = Mockito.mockStatic(EnvFactory.class); + + Fixture() throws Exception { + context.setThreadLocalInfo(); + context.setQueryId(new TUniqueId(1L, 2L)); + EnvFactory factory = Mockito.mock(EnvFactory.class); + factoryMock.when(EnvFactory::getInstance).thenReturn(factory); + Mockito.when(factory.createCoordinator(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong())) + .thenReturn(coordinator); + Mockito.when(transactionManager.getTransaction(101L)).thenReturn(transaction); + + CatalogProperty properties = Mockito.mock(CatalogProperty.class); + Mockito.when(properties.getStoragePropertiesMap()).thenReturn(Collections.emptyMap()); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(catalog.getName()).thenReturn("iceberg"); + Mockito.when(catalog.getCatalogProperty()).thenReturn(properties); + Mockito.when(catalog.getTransactionManager()).thenReturn(transactionManager); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDatabase()).thenReturn(Mockito.mock(DatabaseIf.class)); + Mockito.when(table.getName()).thenReturn("table"); + Mockito.when(table.getDbName()).thenReturn("db"); + + Mockito.when(nativeTable.schema()).thenReturn(new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()))); + Mockito.when(nativeTable.spec()).thenReturn(PartitionSpec.unpartitioned()); + Mockito.when(nativeTable.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(nativeTable.location()).thenReturn("file:///tmp/table"); + Mockito.when(nativeTable.properties()).thenReturn(Collections.singletonMap( + TableProperties.WRITE_DATA_LOCATION, "file:///tmp/custom/data/")); + } + + @Override + public void close() { + factoryMock.close(); + ConnectContext.remove(); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutorTest.java index 99163c19dcb5be..296df158edebdc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutorTest.java @@ -112,6 +112,8 @@ public void testFinalizeSinkAndBeforeExecPropagateRewritableDeleteMetadata() thr executor.txnId = 10L; executor.beforeExec(); + Mockito.verify(transaction).addCommitLocations(tDataSink); + Mockito.verify(transaction).beginDelete(table, targetIcebergTable); ArgumentCaptor>> deleteFilesCaptor = ArgumentCaptor.forClass(Map.class); Mockito.verify(transaction).setRewrittenDeleteFilesByReferencedDataFile(deleteFilesCaptor.capture()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertFailureLifecycleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertFailureLifecycleTest.java index c33d3f8d69854c..a2b221015dd831 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertFailureLifecycleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertFailureLifecycleTest.java @@ -21,17 +21,24 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EnvFactory; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.hive.HiveTransactionMgr; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergMetadataOps; +import org.apache.doris.datasource.iceberg.IcebergTransaction; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.trees.plans.physical.PhysicalSink; import org.apache.doris.planner.DataSink; import org.apache.doris.planner.PlanFragment; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.Coordinator; +import org.apache.doris.qe.QueryState; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.thrift.TDataSink; +import org.apache.doris.thrift.TDataSinkType; +import org.apache.doris.thrift.TIcebergCommitData; +import org.apache.doris.thrift.TIcebergTableSink; import org.apache.doris.thrift.TUniqueId; import org.apache.doris.transaction.GlobalExternalTransactionInfoMgr; import org.apache.doris.transaction.IcebergTransactionManager; @@ -43,6 +50,7 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.util.Collections; import java.util.Optional; class IcebergInsertFailureLifecycleTest { @@ -54,15 +62,21 @@ void tearDown() { @Test void testNormalInsertBeginFailureClearsLocalAndGlobalRegistriesAfterReset() throws Exception { - assertBeginFailureClearsRegistries(false); + assertFailureClearsRegistries(false, false); } @Test void testEmptyInsertBeginFailureClearsLocalAndGlobalRegistriesAfterReset() throws Exception { - assertBeginFailureClearsRegistries(true); + assertFailureClearsRegistries(true, false); } - private void assertBeginFailureClearsRegistries(boolean emptyInsert) throws Exception { + @Test + void testCommitPathRejectionFailsStatementAndClearsRegistries() throws Exception { + // Exercise the completion/error path without a BE; the transaction already has a final report. + assertFailureClearsRegistries(true, true); + } + + private void assertFailureClearsRegistries(boolean emptyInsert, boolean rejectCommitPath) throws Exception { Env env = Mockito.mock(Env.class); GlobalExternalTransactionInfoMgr globalTransactions = new GlobalExternalTransactionInfoMgr(); Mockito.when(env.getNextId()).thenReturn(101L); @@ -99,7 +113,7 @@ private void assertBeginFailureClearsRegistries(boolean emptyInsert) throws Exce factoryMock.when(EnvFactory::getInstance).thenReturn(factory); FailingIcebergExecutor executor = new FailingIcebergExecutor( - context, table, emptyInsert); + context, table, emptyInsert, rejectCommitPath); executor.beginTransaction(); long transactionId = executor.getTxnId(); Assertions.assertNotNull(transactionManager.getTransaction(transactionId)); @@ -112,6 +126,9 @@ private void assertBeginFailureClearsRegistries(boolean emptyInsert) throws Exce executor.executeSingleInsert(stmtExecutor); } + Assertions.assertEquals(QueryState.MysqlStateType.ERR, context.getState().getStateType()); + Assertions.assertTrue(context.getState().getErrorMessage().contains( + rejectCommitPath ? "outside the sink locations" : "catalog runtime changed before begin")); Assertions.assertThrows(UserException.class, () -> transactionManager.getTransaction(transactionId)); Assertions.assertThrows(RuntimeException.class, @@ -122,19 +139,30 @@ private void assertBeginFailureClearsRegistries(boolean emptyInsert) throws Exce } private static class FailingIcebergExecutor extends BaseExternalTableInsertExecutor { + private final boolean rejectCommitPath; + FailingIcebergExecutor(ConnectContext context, IcebergExternalTable table, - boolean emptyInsert) { + boolean emptyInsert, boolean rejectCommitPath) { super(context, table, "label", Mockito.mock(NereidsPlanner.class), Optional.empty(), emptyInsert, -1L); + this.rejectCommitPath = rejectCommitPath; } @Override protected void beforeExec() throws UserException { - throw new UserException("catalog runtime changed before begin"); + if (!rejectCommitPath) { + throw new UserException("catalog runtime changed before begin"); + } } @Override - protected void doBeforeCommit() { + protected void doBeforeCommit() throws UserException { + IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId); + transaction.addCommitLocations(new TDataSink(TDataSinkType.ICEBERG_TABLE_SINK) + .setIcebergTableSink(new TIcebergTableSink().setOriginalOutputPath("oss://bucket/table/data"))); + transaction.updateIcebergCommitData(Collections.singletonList( + new TIcebergCommitData().setFilePath("s3://bucket/table/data/file.parquet"))); + transaction.finishInsert(NameMapping.createForTest("db", "table")); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutorTest.java index 89e85bd104dc76..ed15da92995c24 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutorTest.java @@ -115,6 +115,8 @@ public void testFinalizeSinkAndBeforeExecPropagateRewritableDeleteMetadata() thr executor.txnId = 11L; executor.beforeExec(); + Mockito.verify(transaction).addCommitLocations(tDataSink); + Mockito.verify(transaction).beginMerge( table, targetIcebergTable, java.util.Optional.empty()); ArgumentCaptor>> deleteFilesCaptor = ArgumentCaptor.forClass(Map.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergTableSinkTest.java index bf6d2e6945db1b..fbcc9861a9dd57 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergTableSinkTest.java @@ -23,7 +23,10 @@ import org.apache.doris.datasource.property.storage.OSSProperties; import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext; import org.apache.doris.nereids.trees.plans.commands.insert.IcebergInsertCommandContext; +import org.apache.doris.thrift.TIcebergDeleteSink; +import org.apache.doris.thrift.TIcebergMergeSink; import org.apache.doris.thrift.TIcebergTableSink; import org.apache.iceberg.PartitionSpec; @@ -131,6 +134,41 @@ public void testBindPrefersOssDataPlanePropertiesOverGenericS3() throws Exceptio thriftSink.getHadoopConfig().get("AWS_ENDPOINT")); Assertions.assertEquals("oss-ak", thriftSink.getHadoopConfig().get("AWS_ACCESS_KEY")); Assertions.assertEquals("s3://bucket/table/data", thriftSink.getOriginalOutputPath()); + + // Every writer must receive the same data directory in the storage and metadata forms. + // Custom write.data.path and its trailing slash must also survive DELETE and MERGE planning. + Mockito.when(table.location()).thenReturn("oss://bucket/table"); + Map dataLocations = new HashMap<>(); + dataLocations.put("oss://bucket/table/data", "s3://bucket/table/data"); + dataLocations.put("oss://bucket/custom/data/", "s3://bucket/custom/data"); + for (Map.Entry location : dataLocations.entrySet()) { + String rawLocation = location.getKey(); + String outputPath = location.getValue(); + for (String version : new String[] {"2", "3"}) { + Map tableProperties = new HashMap<>(); + tableProperties.put(TableProperties.FORMAT_VERSION, version); + tableProperties.put(TableProperties.WRITE_DATA_LOCATION, rawLocation); + Mockito.when(table.properties()).thenReturn(tableProperties); + + IcebergTableSink tableSink = new IcebergTableSink(targetTable, table); + tableSink.bindDataSink(Optional.empty()); + Assertions.assertEquals(outputPath, tableSink.tDataSink.getIcebergTableSink().getOutputPath()); + Assertions.assertEquals(rawLocation, tableSink.tDataSink.getIcebergTableSink().getOriginalOutputPath()); + + IcebergDeleteSink deleteSink = new IcebergDeleteSink(targetTable, table, new DeleteCommandContext()); + deleteSink.bindDataSink(Optional.empty()); + TIcebergDeleteSink thriftDeleteSink = deleteSink.tDataSink.getIcebergDeleteSink(); + Assertions.assertEquals(outputPath, thriftDeleteSink.getOutputPath()); + Assertions.assertEquals(rawLocation, thriftDeleteSink.getTableLocation()); + + IcebergMergeSink mergeSink = new IcebergMergeSink(targetTable, table, new DeleteCommandContext(), true); + mergeSink.bindDataSink(Optional.empty()); + TIcebergMergeSink thriftMergeSink = mergeSink.tDataSink.getIcebergMergeSink(); + Assertions.assertEquals(outputPath, thriftMergeSink.getOutputPath()); + Assertions.assertEquals(rawLocation, thriftMergeSink.getOriginalOutputPath()); + Assertions.assertEquals(rawLocation, thriftMergeSink.getTableLocation()); + } + } } @Test diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index cc29b7811eefb3..e5e9391ab787c2 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -437,6 +437,9 @@ struct TIcebergColumnStats { } struct TIcebergCommitData { + // The path Iceberg metadata records for the file: under the table's own form of the location + // the sink carries (TIcebergTableSink.original_output_path, TIcebergDeleteSink.table_location), + // never under the normalized output_path BE wrote it at. 1: optional string file_path // File-level record count used to construct Iceberg data/delete file metadata. 2: optional i64 row_count @@ -489,6 +492,12 @@ struct TIcebergTableSink { 9: optional map hadoop_config 10: optional bool overwrite 11: optional Types.TFileType file_type + // The raw (un-normalized) data location: the same directory as output_path, in the table's + // own form. BE writes data files under output_path and commits them under + // original_output_path, e.g. written to s3://b/db/t/data/p=1/x.parquet, committed as + // oss://b/db/t/data/p=1/x.parquet. "Same directory" is not "same text after the scheme": + // normalization can also rewrite the authority, e.g. abfss://c@acct.dfs.core.windows.net/t/data + // becomes s3://c/t/data. 12: optional string original_output_path 13: optional PlanNodes.TFileCompressType compression_type 14: optional list broker_addresses; @@ -520,6 +529,11 @@ struct TIcebergDeleteSink { 5: optional PlanNodes.TFileFormatType file_format 6: optional PlanNodes.TFileCompressType compress_type 7: optional string output_path + // The raw (un-normalized) data location: the same directory as output_path, in the table's + // own form. BE writes delete files under output_path and commits them under table_location, + // e.g. written to s3://b/db/t/data/delete_x.parquet, committed as oss://b/db/t/data/delete_x.parquet. + // It must not be the table root, or committed paths would miss the data directory. See + // TIcebergTableSink.original_output_path for what "same directory" means. 8: optional string table_location 9: optional map hadoop_config 10: optional Types.TFileType file_type @@ -544,6 +558,8 @@ struct TIcebergMergeSink { 7: optional PlanNodes.TFileFormatType file_format 8: optional PlanNodes.TFileCompressType compression_type 9: optional string output_path + // Same contract as TIcebergTableSink.original_output_path, for the data files; FE sets it to + // the same value as table_location, which the delete files are committed under. 10: optional string original_output_path 11: optional map hadoop_config 12: optional Types.TFileType file_type @@ -557,6 +573,7 @@ struct TIcebergMergeSink { // delete side (position delete only) 20: optional TFileContent delete_type + // Same contract as TIcebergDeleteSink.table_location: the raw form of output_path's directory. 21: optional string table_location 22: optional i32 partition_spec_id_for_delete 23: optional string partition_data_json_for_delete diff --git a/regression-test/data/external_table_p0/iceberg/dml/test_iceberg_update_delete_basic.out.org b/regression-test/data/external_table_p0/iceberg/dml/test_iceberg_update_delete_basic.out.org deleted file mode 100644 index c42c6138a819f6..00000000000000 --- a/regression-test/data/external_table_p0/iceberg/dml/test_iceberg_update_delete_basic.out.org +++ /dev/null @@ -1,77 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !parquet_q01 -- -1 - --- !order_parquet_q02 -- -1 Updated 25 -2 Bob 30 -3 Charlie 35 - --- !parquet_q03 -- -1 - --- !order_parquet_q04 -- -3 Charlie 35 -1 Updated 25 - --- !order_parquet_q05 -- -1 s3://warehouse/wh/test_update_delete_basic_db/test_update_delete_basic_tbl_parquet/data/delete_pos_5066988_0553b2d5.zstd.parquet PARQUET 0 1 2078 \N \N \N \N \N \N \N \N \N \N \N \N \N \N {"age":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "id":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "name":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}} -1 s3://warehouse/wh/test_update_delete_basic_db/test_update_delete_basic_tbl_parquet/data/delete_pos_5066988_3638addc.zstd.parquet PARQUET 0 1 2078 \N \N \N \N \N \N \N \N \N \N \N \N \N \N {"age":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "id":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "name":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}} - --- !parquet_q06 -- -1 - --- !order_parquet_q07 -- -12 Cat 22 2024-01-03 -11 Ben 21 2024-01-02 -10 UpdatedP 20 2024-01-01 - --- !parquet_q08 -- -1 - --- !order_parquet_q09 -- -12 Cat 22 2024-01-03 -10 UpdatedP 20 2024-01-01 - --- !order_parquet_q10 -- -1 s3://warehouse/wh/test_update_delete_basic_db/test_update_delete_basic_tbl_par_parquet/data/delete_pos_5092880_cd626510.zstd.parquet PARQUET 0 {"dt_day":"2024-01-01"} 1 2226 \N \N \N \N \N \N \N \N \N \N \N \N \N \N {"age":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "dt":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "id":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "name":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}} -1 s3://warehouse/wh/test_update_delete_basic_db/test_update_delete_basic_tbl_par_parquet/data/delete_pos_7425526_6dd1173c.zstd.parquet PARQUET 0 {"dt_day":"2024-01-02"} 1 2225 \N \N \N \N \N \N \N \N \N \N \N \N \N \N {"age":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "dt":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "id":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "name":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}} - --- !orc_q01 -- -1 - --- !order_orc_q02 -- -1 Updated 25 -2 Bob 30 -3 Charlie 35 - --- !orc_q03 -- -1 - --- !order_orc_q04 -- -3 Charlie 35 -1 Updated 25 - --- !order_orc_q05 -- -1 s3://warehouse/wh/test_update_delete_basic_db/test_update_delete_basic_tbl_orc/data/delete_pos_4451312_c1ffb646.zlib.orc ORC 0 1 926 \N \N \N \N \N \N \N \N \N \N \N \N \N \N {"age":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "id":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "name":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}} -1 s3://warehouse/wh/test_update_delete_basic_db/test_update_delete_basic_tbl_orc/data/delete_pos_4451312_c5f91315.zlib.orc ORC 0 1 924 \N \N \N \N \N \N \N \N \N \N \N \N \N \N {"age":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "id":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "name":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}} - --- !orc_q06 -- -1 - --- !order_orc_q07 -- -11 Ben 21 2024-01-02 -12 Cat 22 2024-01-03 -10 UpdatedP 20 2024-01-01 - --- !orc_q08 -- -1 - --- !order_orc_q09 -- -12 Cat 22 2024-01-03 -10 UpdatedP 20 2024-01-01 - --- !order_orc_q10 -- -1 s3://warehouse/wh/test_update_delete_basic_db/test_update_delete_basic_tbl_par_orc/data/delete_pos_7724320_d7587f50.zlib.orc ORC 0 {"dt_day":"2024-01-01"} 1 986 \N \N \N \N \N \N \N \N \N \N \N \N \N \N {"age":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "dt":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "id":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "name":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}} -1 s3://warehouse/wh/test_update_delete_basic_db/test_update_delete_basic_tbl_par_orc/data/delete_pos_817404_2d2aa46c.zlib.orc ORC 0 {"dt_day":"2024-01-02"} 1 990 \N \N \N \N \N \N \N \N \N \N \N \N \N \N {"age":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "dt":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "id":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}, "name":{"column_size":null, "value_count":null, "null_value_count":null, "nan_value_count":null, "lower_bound":null, "upper_bound":null}} - diff --git a/regression-test/suites/external_table_p0/iceberg/dml/test_iceberg_update_delete_basic.groovy b/regression-test/suites/external_table_p0/iceberg/dml/test_iceberg_update_delete_basic.groovy index 21384862153c00..9ef8c30b1c81ee 100644 --- a/regression-test/suites/external_table_p0/iceberg/dml/test_iceberg_update_delete_basic.groovy +++ b/regression-test/suites/external_table_p0/iceberg/dml/test_iceberg_update_delete_basic.groovy @@ -47,6 +47,15 @@ suite("test_iceberg_update_delete_basic", "p0,external,iceberg,external_docker,e sql """create database if not exists ${dbName}""" sql """use ${dbName}""" + // Delete files must be recorded under the table's own data location (s3a:// in this fixture), not + // under the s3:// address BE writes them at. + def dataLocationOf = { String table -> + String ddl = sql("""show create table ${table}""")[0][1] + def matcher = ddl =~ /LOCATION '([^']+)'/ + assert matcher.find() : "no LOCATION in ${ddl}" + return matcher.group(1) + "/data/" + } + def formats = ["parquet", "orc"] for (String format : formats) { logger.info("Run update/delete test with format ${format}") @@ -94,10 +103,11 @@ suite("test_iceberg_update_delete_basic", "p0,external,iceberg,external_docker,e FROM ${catalogName}.${dbName}.${formatTableName}\$delete_files """ assert deleteFiles.size() > 0 : "Delete files should be created for ${formatTableName}" + String dataLocation = dataLocationOf(formatTableName) for (def row : deleteFiles) { String filePath = row[0].toString() String fileFormat = row[1].toString() - assert filePath.contains("/data/delete_pos_") + assert filePath.startsWith(dataLocation + "delete_pos_") : "${filePath} is not under ${dataLocation}" assert filePath.endsWith(format == "parquet" ? ".parquet" : ".orc") assert fileFormat.equalsIgnoreCase(format) } @@ -142,10 +152,12 @@ suite("test_iceberg_update_delete_basic", "p0,external,iceberg,external_docker,e FROM ${catalogName}.${dbName}.${formatTableNamePartition}\$delete_files """ assert partitionDeleteFiles.size() > 0 : "Delete files should be created for ${formatTableNamePartition}" + String partitionDataLocation = dataLocationOf(formatTableNamePartition) for (def row : partitionDeleteFiles) { String filePath = row[0].toString() String fileFormat = row[1].toString() - assert filePath.contains("/data/delete_pos_") + assert filePath.startsWith(partitionDataLocation + "delete_pos_") : + "${filePath} is not under ${partitionDataLocation}" assert filePath.endsWith(format == "parquet" ? ".parquet" : ".orc") assert fileFormat.equalsIgnoreCase(format) } diff --git a/regression-test/suites/external_table_p0/iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md b/regression-test/suites/external_table_p0/iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md index 6829fd1d4c4fa6..c41458289a0593 100644 --- a/regression-test/suites/external_table_p0/iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md +++ b/regression-test/suites/external_table_p0/iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md @@ -53,6 +53,7 @@ under the License. | R16 | CTAS 对复杂类型、NULL、分区 transform、文件格式和失败清理的行为不一致 | 黑盒:DDL + writer 一体提交 | schema、文件格式、原子性 | P0 | | R17 | sort order、distribution mode、多次文件 flush 和并发 commit 组合导致乱序、丢行或重复提交 | 白盒:exchange、sort writer、optimistic commit | 分布式正确性、稳定性 | P0 | | R18 | STRING identity/bucket/truncate 对空串、中文、emoji、组合字符和 NULL 的物理分区值计算错误 | 黑盒:UTF-8 transform metadata | 分区路由、裁剪 | P0 | +| R19 | 写出的 data file、position delete 或 deletion vector 在元数据中记录为 BE 使用的 normalized 地址(如 s3://),而非表自身的位置(如 s3a://、oss://) | 白盒:sink 写入路径与提交路径 | 其他引擎和维护操作能否打开文件 | P0 | ## 组合覆盖 @@ -87,6 +88,7 @@ under the License. | 排序与分布属性 | 多列 sort order、NULL ordering、none/hash/range distribution、强制多文件 flush | 已覆盖(验证通过) | `test_iceberg_write_order_distribution_properties` | | 并发写入 | 同行冲突 MERGE 的串行化不变量、非冲突分布式 append | 已覆盖(验证通过) | `test_iceberg_write_concurrent_merge_invariants` | | 分布式执行 | 多 bucket 源表、多分区 Iceberg sink、多 BE writer、suite 间无共享 catalog/database | 已覆盖(验证通过) | 所有本次新增 suite | +| 文件路径记录 | INSERT、分区 OVERWRITE、branch INSERT、DELETE、UPDATE、MERGE 和 rewrite_data_files 写出的所有文件(v2 position delete、v3 deletion vector)均记录在表的 data location 下 | 已覆盖(常规回归) | `test_iceberg_write_file_locations` | | Spark 交叉验证 | 十五个正向 suite 在最后一次写入后逐行比较;STRING transform suite 另行比较演进前后的物理分区 metadata | 已覆盖(验证通过) | 十五个正向 suite;物理 metadata 见 `test_iceberg_write_string_transform_metadata` | ## 本次新增用例设计 @@ -112,9 +114,10 @@ under the License. | W17 | 验证 sort order、distribution mode 和多文件 flush | R03、R11、R17 | 正确性、稳定性 | 多 BE Doris | NULL sort key、多列升降序、none/hash/range、低 target file size | 计划包含声明排序,多文件总行数正确,三种分布模式结果一致 | | W18 | 验证并发 MERGE 与 append 的提交不变量 | R11、R13、R17 | 并发、原子性 | 多 BE Doris | readiness barrier 保证两个独立会话同时具备 dispatch 条件;同行更新与互不冲突 append | 同行提交可串行化且基数为一;仅接受 Iceberg validation/commit conflict;非冲突写入无丢失或重复 | | W19 | 验证 MERGE source projection 进入 truncate transform 的类型安全 | R12、R18 | 常规回归、稳定性 | Iceberg v2 MOR | required STRING truncate 列执行匹配更新与未匹配插入 | MERGE 接受 nullable 物理投影中的非 NULL 值,并生成正确物理分区 | +| W20 | 验证各写入操作的文件路径均记录在表自身位置下 | R19 | 正确性、兼容性 | REST fixture 的 warehouse 为 s3a://,BE 将其规范化为 s3:// | 在 v2 和 v3 表上依次执行 INSERT、分区 OVERWRITE、branch INSERT、DELETE、UPDATE、MERGE 和 rewrite_data_files | `$all_files` 中每个文件均以 `SHOW CREATE TABLE` 的 LOCATION 加 `/data/` 为前缀;新增写入操作也加入此用例 | ## P0 覆盖检查 -R01-R18 均映射到至少一个 P0 regression。十五个正向 suite 在最后一次成功写入后均由 Spark/Doris 交叉校验同表逻辑结果;transform、rollover、CTAS property 和失败原子性另有物理 metadata 或状态 oracle。W08、W10、W19 已作为常规回归运行;仍可能写出不可读 required-null 文件的 W07 保留隔离开关,避免破坏共享集群。 +R01-R19 均映射到至少一个 P0 regression。十五个正向 suite 在最后一次成功写入后均由 Spark/Doris 交叉校验同表逻辑结果;transform、rollover、CTAS property 和失败原子性另有物理 metadata 或状态 oracle。W08、W10、W19 已作为常规回归运行;仍可能写出不可读 required-null 文件的 W07 保留隔离开关,避免破坏共享集群。 本矩阵未覆盖项为 0。COW 行级 DML、branch 行级 DML、tag 写入和 Avro 写入属于当前明确能力边界,均以预期拒绝用例固化错误语义与失败原子性;仅 required-null 已知缺陷继续使用隔离负向 regression。 diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_file_locations.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_file_locations.groovy new file mode 100644 index 00000000000000..b8cc5203d6c6e6 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_file_locations.groovy @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Every file Doris writes into an Iceberg table must be recorded under the table's own location, not +// under the address BE wrote it at. The REST fixture's warehouse is s3a://warehouse/wh/, which FE +// normalizes to s3:// for BE, so a writer that records the s3:// address is caught here: delete files +// and deletion vectors were once committed that way. Each write operation Doris supports runs below; +// a new one belongs here too. +suite("test_iceberg_write_file_locations", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_file_locations" + String dbName = "iceberg_write_file_locations_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + // The table's data location, taken from its metadata rather than from any file Doris wrote. + def dataLocationOf = { String table -> + String ddl = sql("""show create table ${table}""")[0][1] + def matcher = ddl =~ /LOCATION '([^']+)'/ + assertTrue(matcher.find(), "no LOCATION in ${ddl}") + return matcher.group(1) + "/data/" + } + + // Every file of every snapshot, branches included: data files, position deletes and deletion vectors. + def assertFilesUnderDataLocation = { String table, String operation -> + String dataLocation = dataLocationOf(table) + def files = sql """select content, file_format, file_path from ${table}\$all_files""" + assertTrue(files.size() > 0, "no files in ${table} after ${operation}") + for (def file : files) { + String path = file[2].toString() + assertTrue(path.startsWith(dataLocation), + "${operation} recorded ${file[1]} file ${path} (content ${file[0]}) outside ${dataLocation}") + } + } + + def countFiles = { String table, String predicate -> + return (sql """select count(*) from ${table}\$all_files where ${predicate}""")[0][0] as long + } + + // Format v2: data files and position delete files. + sql """drop table if exists file_locations_v2""" + sql """ + create table file_locations_v2 ( + id int, + region string, + payload string + ) + partition by list (region) () + properties ( + "format-version" = "2", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + sql """insert into file_locations_v2 values (1, 'A', 'a1'), (2, 'B', 'b1'), (3, 'C', 'c1')""" + assertFilesUnderDataLocation("file_locations_v2", "INSERT") + sql """insert into file_locations_v2 values (4, 'A', 'a2'), (5, 'B', 'b2')""" + assertFilesUnderDataLocation("file_locations_v2", "second INSERT") + sql """insert overwrite table file_locations_v2 partition (region = 'C') values (6, 'c2')""" + assertFilesUnderDataLocation("file_locations_v2", "INSERT OVERWRITE PARTITION") + sql """alter table file_locations_v2 create branch location_branch""" + sql """insert into file_locations_v2@branch(location_branch) values (7, 'A', 'branch')""" + assertFilesUnderDataLocation("file_locations_v2", "branch INSERT") + sql """delete from file_locations_v2 where id = 1""" + assertFilesUnderDataLocation("file_locations_v2", "DELETE") + sql """update file_locations_v2 set payload = 'b1-updated' where id = 2""" + assertFilesUnderDataLocation("file_locations_v2", "UPDATE") + sql """ + merge into file_locations_v2 t + using ( + select 4 as id, 'A' as region, 'a2-merged' as payload + union all + select 8, 'D', 'd1' + ) s + on t.id = s.id + when matched then update set payload = s.payload + when not matched then insert (id, region, payload) values (s.id, s.region, s.payload) + """ + assertFilesUnderDataLocation("file_locations_v2", "MERGE") + sql """ + alter table ${catalogName}.${dbName}.file_locations_v2 + execute rewrite_data_files("target-file-size-bytes" = "10485760", "min-input-files" = "2") + """ + assertFilesUnderDataLocation("file_locations_v2", "rewrite_data_files") + assertTrue(countFiles("file_locations_v2", "content = 1") > 0, "DELETE / UPDATE / MERGE wrote no delete files") + + // Format v3: row-level operations write deletion vectors into Puffin files. + sql """drop table if exists file_locations_v3""" + sql """ + create table file_locations_v3 ( + id int, + region string, + payload string + ) + partition by list (region) () + properties ( + "format-version" = "3", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + sql """insert into file_locations_v3 values (1, 'A', 'a1'), (2, 'B', 'b1'), (3, 'C', 'c1')""" + assertFilesUnderDataLocation("file_locations_v3", "INSERT") + sql """delete from file_locations_v3 where id = 1""" + assertFilesUnderDataLocation("file_locations_v3", "DELETE") + sql """update file_locations_v3 set payload = 'b1-updated' where id = 2""" + assertFilesUnderDataLocation("file_locations_v3", "UPDATE") + sql """ + merge into file_locations_v3 t + using ( + select 3 as id, 'C' as region, 'c1-merged' as payload + union all + select 4, 'D', 'd1' + ) s + on t.id = s.id + when matched then update set payload = s.payload + when not matched then insert (id, region, payload) values (s.id, s.region, s.payload) + """ + assertFilesUnderDataLocation("file_locations_v3", "MERGE") + assertTrue(countFiles("file_locations_v3", "upper(file_format) = 'PUFFIN'") > 0, + "DELETE / UPDATE / MERGE wrote no deletion vectors") +}