diff --git a/be/src/exec/partitioner/external/external_partition_function_factory.cpp b/be/src/exec/partitioner/external/external_partition_function_factory.cpp index ed87a971788e59..5b31290076239e 100644 --- a/be/src/exec/partitioner/external/external_partition_function_factory.cpp +++ b/be/src/exec/partitioner/external/external_partition_function_factory.cpp @@ -21,6 +21,7 @@ #include "common/status.h" #include "exec/partitioner/external/paimon_fixed_bucket_partition_function.h" +#include "format/transformer/iceberg_partition_function.h" namespace doris { namespace { @@ -48,6 +49,19 @@ Status parse_indexes(std::string_view value, std::vector* const indexes return Status::OK(); } +Status create_paimon_fixed_bucket_function( + PartitionerBase::HashValType logical_partition_count, int32_t num_buckets, + std::vector partition_indexes, std::vector bucket_indexes, + const std::vector& partition_exprs, + std::unique_ptr* const partition_function) { + auto function = std::make_unique( + logical_partition_count, num_buckets, std::move(partition_indexes), + std::move(bucket_indexes)); + RETURN_IF_ERROR(function->init(partition_exprs)); + *partition_function = std::move(function); + return Status::OK(); +} + Status create_paimon_fixed_bucket_function( const TExternalTableSinkHashPartitionInfo& partition_info, PartitionerBase::HashValType logical_partition_count, @@ -69,14 +83,77 @@ Status create_paimon_fixed_bucket_function( RETURN_IF_ERROR(parse_int32(options.at("num_buckets"), &num_buckets)); RETURN_IF_ERROR(parse_indexes(options.at("partition_field_indexes"), &partition_indexes)); RETURN_IF_ERROR(parse_indexes(options.at("bucket_field_indexes"), &bucket_indexes)); - auto function = std::make_unique( + RETURN_IF_ERROR(create_paimon_fixed_bucket_function( logical_partition_count, num_buckets, std::move(partition_indexes), - std::move(bucket_indexes)); - RETURN_IF_ERROR(function->init(partition_exprs)); - *partition_function = std::move(function); + std::move(bucket_indexes), partition_exprs, partition_function)); return Status::OK(); } +Status create_legacy_external_partition_function( + const TExternalTableSinkHashPartitionInfo& partition_info, + PartitionerBase::HashValType logical_partition_count, ShuffleHashMethod hash_method, + const std::vector& partition_exprs, + std::unique_ptr* const partition_function) { + if (!partition_info.__isset.algorithm) { + return Status::InvalidArgument("External sink partition function is missing"); + } + switch (partition_info.algorithm) { + case TExternalTableSinkHashAlgorithm::DIRECT_HASH: { + if (partition_info.__isset.partition_transforms || + partition_info.__isset.paimon_fixed_bucket_info) { + return Status::InvalidArgument( + "Legacy direct external sink hash contains incompatible metadata"); + } + auto function = + std::make_unique(logical_partition_count, hash_method); + RETURN_IF_ERROR(function->init(partition_exprs)); + *partition_function = std::move(function); + return Status::OK(); + } + case TExternalTableSinkHashAlgorithm::ICEBERG_TRANSFORM: { + if (partition_info.__isset.paimon_fixed_bucket_info) { + return Status::InvalidArgument( + "Legacy Iceberg sink routing contains incompatible Paimon metadata"); + } + if (!partition_info.__isset.partition_transforms || + partition_info.partition_transforms.size() != partition_exprs.size()) { + return Status::InvalidArgument( + "Legacy Iceberg sink partition transforms must match expression count"); + } + std::vector fields; + fields.reserve(partition_exprs.size()); + for (size_t index = 0; index < partition_exprs.size(); ++index) { + TIcebergPartitionField field; + field.__set_transform(partition_info.partition_transforms[index]); + field.__set_source_expr(partition_exprs[index]); + fields.emplace_back(std::move(field)); + } + auto function = std::make_unique( + logical_partition_count, hash_method, std::vector {}, std::move(fields)); + RETURN_IF_ERROR(function->init({})); + *partition_function = std::move(function); + return Status::OK(); + } + case TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET: { + if (partition_info.__isset.partition_transforms) { + return Status::InvalidArgument( + "Legacy Paimon fixed-bucket routing contains incompatible Iceberg metadata"); + } + if (!partition_info.__isset.paimon_fixed_bucket_info) { + return Status::InvalidArgument( + "Legacy Paimon fixed-bucket routing metadata is missing"); + } + const auto& info = partition_info.paimon_fixed_bucket_info; + return create_paimon_fixed_bucket_function( + logical_partition_count, info.num_buckets, info.partition_field_indexes, + info.bucket_field_indexes, partition_exprs, partition_function); + } + default: + return Status::InvalidArgument("Unsupported legacy external sink hash algorithm {}", + static_cast(partition_info.algorithm)); + } +} + } // namespace Status create_external_partition_function( @@ -87,6 +164,11 @@ Status create_external_partition_function( if (partition_function == nullptr) { return Status::InvalidArgument("External partition function output is null"); } + if (!partition_info.__isset.partition_function) { + return create_legacy_external_partition_function(partition_info, logical_partition_count, + hash_method, partition_exprs, + partition_function); + } if (partition_info.partition_function == "paimon_fixed_bucket") { return create_paimon_fixed_bucket_function(partition_info, logical_partition_count, partition_exprs, partition_function); diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index c2bf920c8ae804..2611b47ce3deb9 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -2602,7 +2602,7 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r } const bool requires_external_file_ack = - params.__isset.iceberg_commit_datas || params.__isset.connector_commit_data; + params.__isset.iceberg_commit_datas || params.__isset.paimon_commit_messages; if (rpc_status.ok() && requires_external_file_ack && (!res.__isset.external_file_commit_data_accepted || !res.external_file_commit_data_accepted)) { diff --git a/be/src/exec/sink/viceberg_delete_sink.cpp b/be/src/exec/sink/viceberg_delete_sink.cpp index 92d26b560f7160..55b732d6527298 100644 --- a/be/src/exec/sink/viceberg_delete_sink.cpp +++ b/be/src/exec/sink/viceberg_delete_sink.cpp @@ -34,7 +34,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "exec/common/endian.h" -#include "exec/sink/writer/iceberg/iceberg_writer_compatibility.h" +#include "exec/sink/writer/external_file_writer_compatibility.h" #include "exprs/vexpr.h" #include "format/table/deletion_vector.h" #include "format/table/iceberg_delete_file_reader_helper.h" @@ -204,7 +204,7 @@ Status VIcebergDeleteSink::init_properties(ObjectPool* pool) { Status VIcebergDeleteSink::open(RuntimeState* state, RuntimeProfile* profile) { _state = state; - RETURN_IF_ERROR(validate_iceberg_external_file_report_ack(state->query_options())); + RETURN_IF_ERROR(validate_external_file_report_ack(state->query_options())); // Initialize counters _written_rows_counter = ADD_COUNTER(profile, "RowsWritten", TUnit::UNIT); diff --git a/be/src/exec/sink/writer/iceberg/iceberg_writer_compatibility.h b/be/src/exec/sink/writer/external_file_writer_compatibility.h similarity index 85% rename from be/src/exec/sink/writer/iceberg/iceberg_writer_compatibility.h rename to be/src/exec/sink/writer/external_file_writer_compatibility.h index ae06f4847a4c82..3cd4fd0b0e9ba3 100644 --- a/be/src/exec/sink/writer/iceberg/iceberg_writer_compatibility.h +++ b/be/src/exec/sink/writer/external_file_writer_compatibility.h @@ -22,12 +22,13 @@ namespace doris { -inline Status validate_iceberg_external_file_report_ack(const TQueryOptions& query_options) { +inline Status validate_external_file_report_ack(const TQueryOptions& query_options) { if (!query_options.__isset.supports_external_file_report_ack || !query_options.supports_external_file_report_ack) { // A pre-ACK coordinator cannot safely take ownership of files created by this sink. return Status::NotSupported( - "Iceberg writes require a coordinator that acknowledges external-file reports"); + "External-file writes require a coordinator that acknowledges external-file " + "reports"); } return Status::OK(); } diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp index 8a6c3b35e86acd..c047f5fc88efe6 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -31,8 +31,8 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_struct.h" #include "core/data_type_serde/data_type_serde.h" +#include "exec/sink/writer/external_file_writer_compatibility.h" #include "exec/sink/writer/iceberg/iceberg_partition_path.h" -#include "exec/sink/writer/iceberg/iceberg_writer_compatibility.h" #include "exec/sink/writer/iceberg/partition_transformers.h" #include "exec/sink/writer/iceberg/viceberg_partition_writer.h" #include "exec/sink/writer/iceberg/viceberg_sort_writer.h" @@ -59,7 +59,7 @@ Status VIcebergTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { _state = state; _operator_profile = profile; - RETURN_IF_ERROR(validate_iceberg_external_file_report_ack(state->query_options())); + RETURN_IF_ERROR(validate_external_file_report_ack(state->query_options())); // Get target file size from query options // If value is 0 or not set, use config::iceberg_sink_max_file_size diff --git a/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp b/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp index 7419f88921da33..ae8702db99978e 100644 --- a/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp +++ b/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp @@ -21,9 +21,9 @@ #include "common/logging.h" #include "core/block/block.h" #include "core/block/materialize_block.h" +#include "exec/sink/writer/external_file_writer_compatibility.h" #include "exprs/vexpr_context.h" #include "runtime/runtime_state.h" -#include "util/thrift_util.h" namespace doris { @@ -35,6 +35,8 @@ PaimonTableWriter::PaimonTableWriter(TDataSink t_sink, const VExprContextSPtrs& Status PaimonTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { _state = state; + RETURN_IF_ERROR(validate_external_file_report_ack(state->query_options())); + // Register profile counters _written_rows_counter = ADD_COUNTER(profile, "WrittenRows", TUnit::UNIT); _written_bytes_counter = ADD_COUNTER(profile, "WrittenBytes", TUnit::BYTES); @@ -112,6 +114,16 @@ Status PaimonTableWriter::close(Status status) { } } + // Reserve the shared report budget while the writer can still abort its prepared files. + ExternalFileReportReservation commit_message_reservation; + if (status.ok()) { + Status reserve_st = + _state->reserve_paimon_commit_messages(messages, &commit_message_reservation); + if (!reserve_st.ok()) { + status = reserve_st; + } + } + // If prepare_commit failed or the incoming status was already an error, // abort the writer to clean up uncommitted data files. if (!status.ok()) { @@ -148,14 +160,11 @@ Status PaimonTableWriter::close(Status status) { DORIS_CHECK(msg.__isset.payload); COUNTER_UPDATE(_commit_payload_bytes_counter, static_cast(msg.payload.size())); } - ThriftSerializer serializer(false, 256); - for (auto& message : messages) { - std::string commit_data; - RETURN_IF_ERROR(serializer.serialize(&message, &commit_data)); - RETURN_IF_ERROR(_state->add_connector_commit_data(std::move(commit_data))); - } - if (!messages.empty()) { - LOG(INFO) << "Paimon writer closed: " << messages.size() + const size_t message_count = messages.size(); + _state->publish_reserved_paimon_commit_messages(std::move(messages), + &commit_message_reservation); + if (message_count > 0) { + LOG(INFO) << "Paimon writer closed: " << message_count << " commit messages, total rows=" << _written_rows; } } diff --git a/be/src/exec/sink/writer/paimon/paimon_table_writer.h b/be/src/exec/sink/writer/paimon/paimon_table_writer.h index 7c3d7268b8c0d1..ef5d4002e0fa0b 100644 --- a/be/src/exec/sink/writer/paimon/paimon_table_writer.h +++ b/be/src/exec/sink/writer/paimon/paimon_table_writer.h @@ -60,7 +60,8 @@ class RuntimeState; /// Commit flow (BE only prepares messages; FE is the commit coordinator): /// close() → writer->prepare_commit() /// → collect TPaimonCommitMessage[] (DPCM-framed serialized messages) -/// → serialize each message into RuntimeState::add_connector_commit_data() +/// → reserve report budget while abort is still possible +/// → RuntimeState::publish_reserved_paimon_commit_messages() /// → RPC to FE Coordinator → PaimonTransaction class PaimonTableWriter final { public: diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 049d6b165c9012..fef23162652205 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -57,6 +57,13 @@ #include "util/uid_util.h" namespace doris { + +ExternalFileReportReservation::~ExternalFileReportReservation() { + if (_state != nullptr) { + _state->release_external_file_report_reservation(_reserved_bytes); + } +} + using namespace ErrorCode; Status RuntimeState::add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_data) { @@ -84,25 +91,56 @@ Status RuntimeState::add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_ return Status::OK(); } -Status RuntimeState::add_connector_commit_data(std::string commit_data) { +Status RuntimeState::reserve_paimon_commit_messages( + const std::vector& commit_messages, + ExternalFileReportReservation* reservation) { + DORIS_CHECK(reservation != nullptr); + DORIS_CHECK(reservation->_state == nullptr); + size_t serialized_bytes = 0; + ThriftSerializer serializer(false, 256); + for (const auto& commit_message : commit_messages) { + uint32_t serialized_size = 0; + uint8_t* buffer = nullptr; + RETURN_IF_ERROR(serializer.serialize(&commit_message, &serialized_size, &buffer)); + serialized_bytes += serialized_size + sizeof(uint32_t); + } + constexpr size_t report_envelope_headroom = 1024 * 1024; const size_t thrift_limit = coordinator_thrift_message_limit(); const size_t commit_data_limit = thrift_limit > report_envelope_headroom ? thrift_limit - report_envelope_headroom : 0; std::lock_guard budget_lock(_external_file_report_state->mutex); - if (_external_file_report_state->serialized_commit_bytes + commit_data.size() + - sizeof(uint32_t) > + if (_external_file_report_state->serialized_commit_bytes + serialized_bytes > commit_data_limit) { return Status::InternalError( - "Connector commit metadata exceeds the Thrift report limit; reduce commit " - "metadata size"); + "Paimon commit metadata exceeds the Thrift report limit; reduce output file " + "count"); } - std::lock_guard data_lock(_connector_commit_data_mutex); - _external_file_report_state->serialized_commit_bytes += commit_data.size() + sizeof(uint32_t); - _connector_commit_data.emplace_back(std::move(commit_data)); + _external_file_report_state->serialized_commit_bytes += serialized_bytes; + reservation->_state = this; + reservation->_reserved_bytes = serialized_bytes; return Status::OK(); } +void RuntimeState::publish_reserved_paimon_commit_messages( + std::vector commit_messages, + ExternalFileReportReservation* reservation) { + DORIS_CHECK(reservation != nullptr); + DORIS_CHECK(reservation->_state == this); + std::lock_guard data_lock(_paimon_commit_messages_mutex); + for (auto& commit_message : commit_messages) { + _paimon_commit_messages.emplace_back(std::move(commit_message)); + } + reservation->_state = nullptr; + reservation->_reserved_bytes = 0; +} + +void RuntimeState::release_external_file_report_reservation(size_t reserved_bytes) { + std::lock_guard budget_lock(_external_file_report_state->mutex); + DORIS_CHECK_GE(_external_file_report_state->serialized_commit_bytes, reserved_bytes); + _external_file_report_state->serialized_commit_bytes -= reserved_bytes; +} + size_t RuntimeState::coordinator_thrift_message_limit() const { int32_t effective_thrift_limit = std::max(config::thrift_max_message_size, 0); if (_query_options.__isset.coordinator_thrift_max_message_size && @@ -134,9 +172,9 @@ void RuntimeState::append_external_file_commit_data(TReportExecStatusParams* par params->mc_commit_datas.insert(params->mc_commit_datas.end(), commit_datas.begin(), commit_datas.end()); } - append_connector_commit_data(¶ms->connector_commit_data); - if (!params->connector_commit_data.empty()) { - params->__isset.connector_commit_data = true; + append_paimon_commit_messages(¶ms->paimon_commit_messages); + if (!params->paimon_commit_messages.empty()) { + params->__isset.paimon_commit_messages = true; } } diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index def24619e98973..390365ea31024a 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -78,6 +78,24 @@ class TaskExecutionContext; // Keep RuntimeState self-contained without importing the full frontend Thrift service header. class TReportExecStatusParams; +class RuntimeState; + +// Owns a share of the external-file report budget until the prepared commit messages are +// published. Destruction releases an unpublished reservation automatically. +class ExternalFileReportReservation { +public: + ExternalFileReportReservation() = default; + ~ExternalFileReportReservation(); + + ExternalFileReportReservation(const ExternalFileReportReservation&) = delete; + ExternalFileReportReservation& operator=(const ExternalFileReportReservation&) = delete; + +private: + friend class RuntimeState; + + RuntimeState* _state = nullptr; + size_t _reserved_bytes = 0; +}; class ExternalFileReportState { friend class RuntimeState; @@ -547,11 +565,18 @@ class RuntimeState { Status add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_data); - Status add_connector_commit_data(std::string commit_data); + // Reserve report space before closing the Paimon writer, then either publish the prepared + // messages after a clean close or let the reservation release itself while aborting the writer. + Status reserve_paimon_commit_messages(const std::vector& commit_messages, + ExternalFileReportReservation* reservation); + + void publish_reserved_paimon_commit_messages(std::vector commit_messages, + ExternalFileReportReservation* reservation); - void append_connector_commit_data(std::vector* output) const { - std::lock_guard lock(_connector_commit_data_mutex); - output->insert(output->end(), _connector_commit_data.begin(), _connector_commit_data.end()); + void append_paimon_commit_messages(std::vector* output) const { + std::lock_guard lock(_paimon_commit_messages_mutex); + output->insert(output->end(), _paimon_commit_messages.begin(), + _paimon_commit_messages.end()); } size_t coordinator_thrift_message_limit() const; @@ -913,6 +938,10 @@ class RuntimeState { void merge_register_runtime_filter(const std::set& runtime_filter_ids); private: + friend class ExternalFileReportReservation; + + void release_external_file_report_reservation(size_t reserved_bytes); + Status create_error_log_file(); static const int DEFAULT_BATCH_SIZE = 4062; @@ -1019,8 +1048,8 @@ class RuntimeState { mutable std::mutex _mc_commit_datas_mutex; std::vector _mc_commit_datas; - mutable std::mutex _connector_commit_data_mutex; - std::vector _connector_commit_data; + mutable std::mutex _paimon_commit_messages_mutex; + std::vector _paimon_commit_messages; std::vector> _op_id_to_local_state; diff --git a/be/test/exec/partitioner/external_partition_function_factory_test.cpp b/be/test/exec/partitioner/external_partition_function_factory_test.cpp index d34e88aac1f683..e03265f683092e 100644 --- a/be/test/exec/partitioner/external_partition_function_factory_test.cpp +++ b/be/test/exec/partitioner/external_partition_function_factory_test.cpp @@ -21,8 +21,29 @@ #include #include #include +#include + +#include "runtime/descriptor_helper.h" namespace doris { +namespace { + +TExpr int_slot_ref_expr() { + TExprNode node; + node.__set_node_type(TExprNodeType::SLOT_REF); + node.__set_num_children(0); + node.__set_type(create_type_desc(TYPE_INT)); + node.__set_is_nullable(false); + TSlotRef slot_ref; + slot_ref.__set_slot_id(1); + slot_ref.__set_tuple_id(1); + node.__set_slot_ref(slot_ref); + TExpr expression; + expression.nodes.emplace_back(std::move(node)); + return expression; +} + +} // namespace TEST(ExternalPartitionFunctionFactoryTest, CreatesGenericDirectHash) { TExternalTableSinkHashPartitionInfo info; @@ -78,4 +99,34 @@ TEST(ExternalPartitionFunctionFactoryTest, PaimonFixedBucketValidatesOptions) { EXPECT_NE(status.to_string().find("requires num_buckets"), std::string::npos); } +TEST(ExternalPartitionFunctionFactoryTest, ReadsBranch41DirectHashEncoding) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::DIRECT_HASH); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::SKEWED); + std::unique_ptr function; + + ASSERT_TRUE(create_external_partition_function(info, 3, ShuffleHashMethod::CRC32, {}, &function) + .ok()); + ASSERT_NE(function, nullptr); + EXPECT_EQ(function->partition_count(), 3); +} + +TEST(ExternalPartitionFunctionFactoryTest, ReadsBranch41PaimonFixedBucketEncoding) { + TPaimonFixedBucketInfo fixed_bucket_info; + fixed_bucket_info.__set_num_buckets(4); + fixed_bucket_info.__set_partition_field_indexes({}); + fixed_bucket_info.__set_bucket_field_indexes({0}); + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + info.__set_paimon_fixed_bucket_info(fixed_bucket_info); + std::unique_ptr function; + + ASSERT_TRUE(create_external_partition_function(info, 3, ShuffleHashMethod::CRC32, + {int_slot_ref_expr()}, &function) + .ok()); + ASSERT_NE(function, nullptr); + EXPECT_EQ(function->partition_count(), 3); +} + } // namespace doris diff --git a/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp b/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp index d543c18282e1fe..6685e9e3bc46b7 100644 --- a/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp +++ b/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp @@ -19,10 +19,14 @@ #include +#include +#include + #include "agent/be_exec_version_manager.h" #include "exec/operator/paimon_table_sink_operator.h" #include "exec/sink/writer/paimon/ffi_paimon_write_backend.h" #include "exec/sink/writer/paimon/jni_paimon_write_backend.h" +#include "exec/sink/writer/paimon/paimon_table_writer.h" #include "testutil/mock/mock_runtime_state.h" namespace doris { @@ -87,4 +91,33 @@ TEST(PaimonTableSinkOperatorTest, InitializesAsBlockingSink) { EXPECT_TRUE(sink_operator.prepare(&state).ok()); } +TEST(PaimonTableWriterTest, RejectsCoordinatorWithoutExternalFileReportAck) { + TPaimonTableSink paimon_sink; + TDataSink data_sink; + data_sink.__set_type(TDataSinkType::PAIMON_TABLE_SINK); + data_sink.__set_paimon_table_sink(paimon_sink); + VExprContextSPtrs output_exprs; + PaimonTableWriter writer(std::move(data_sink), output_exprs); + RuntimeState state; + RuntimeProfile profile("test"); + + Status status = writer.open(&state, &profile); + + EXPECT_TRUE(status.is()); + EXPECT_NE(std::string::npos, status.to_string().find("acknowledges external-file reports")); +} + +TEST(PaimonTableWriterTest, ClosesCancelledWriterBeforeOpen) { + TPaimonTableSink paimon_sink; + TDataSink data_sink; + data_sink.__set_type(TDataSinkType::PAIMON_TABLE_SINK); + data_sink.__set_paimon_table_sink(paimon_sink); + VExprContextSPtrs output_exprs; + PaimonTableWriter writer(std::move(data_sink), output_exprs); + + Status status = writer.close(Status::Cancelled("cancelled before open")); + + EXPECT_TRUE(status.is()) << status.to_string(); +} + } // namespace doris diff --git a/be/test/runtime/runtime_state_block_budget_test.cpp b/be/test/runtime/runtime_state_block_budget_test.cpp index 9e7dca7fdbe29c..8f5a4706f87efa 100644 --- a/be/test/runtime/runtime_state_block_budget_test.cpp +++ b/be/test/runtime/runtime_state_block_budget_test.cpp @@ -60,28 +60,57 @@ TEST(RuntimeStateIcebergCommitDataTest, SharesTheReportBudgetAcrossParallelTasks EXPECT_FALSE(second_status.ok()); } -TEST(RuntimeStateIcebergCommitDataTest, SharesTheReportBudgetWithOpaqueConnectorData) { +TEST(RuntimeStateIcebergCommitDataTest, SharesTheReportBudgetWithPaimonCommitData) { RuntimeState iceberg_state; - RuntimeState connector_state; + RuntimeState paimon_state; auto budget = std::make_shared(); iceberg_state.set_external_file_report_state(budget); - connector_state.set_external_file_report_state(budget); + paimon_state.set_external_file_report_state(budget); const int32_t saved_limit = config::thrift_max_message_size; config::thrift_max_message_size = 1024 * 1024 + 512; TIcebergCommitData iceberg_data; iceberg_data.__set_file_path(std::string(300, 'x')); + TPaimonCommitMessage paimon_message; + paimon_message.__set_payload(std::string(300, 'y')); Status first_status = iceberg_state.add_iceberg_commit_datas(iceberg_data); - Status second_status = connector_state.add_connector_commit_data(std::string(300, 'y')); + ExternalFileReportReservation paimon_reservation; + Status second_status = + paimon_state.reserve_paimon_commit_messages({paimon_message}, &paimon_reservation); config::thrift_max_message_size = saved_limit; EXPECT_TRUE(first_status.ok()) << first_status; EXPECT_FALSE(second_status.ok()); - std::vector collected; - connector_state.append_connector_commit_data(&collected); + std::vector collected; + paimon_state.append_paimon_commit_messages(&collected); EXPECT_TRUE(collected.empty()); } +TEST(RuntimeStateIcebergCommitDataTest, ReleasesUnpublishedPaimonReservationOnScopeExit) { + RuntimeState state; + const int32_t saved_limit = config::thrift_max_message_size; + config::thrift_max_message_size = 1024 * 1024 + 512; + TPaimonCommitMessage first_message; + first_message.__set_payload(std::string(300, 'x')); + { + ExternalFileReportReservation first_reservation; + ASSERT_TRUE(state.reserve_paimon_commit_messages({first_message}, &first_reservation).ok()); + } + + TPaimonCommitMessage second_message; + second_message.__set_payload(std::string(300, 'y')); + ExternalFileReportReservation second_reservation; + Status second_status = + state.reserve_paimon_commit_messages({second_message}, &second_reservation); + config::thrift_max_message_size = saved_limit; + ASSERT_TRUE(second_status.ok()) << second_status; + state.publish_reserved_paimon_commit_messages({std::move(second_message)}, &second_reservation); + std::vector collected; + state.append_paimon_commit_messages(&collected); + ASSERT_EQ(1, collected.size()); + EXPECT_EQ(std::string(300, 'y'), collected[0].payload); +} + TEST(RuntimeStateIcebergCommitDataTest, UsesTheSmallerCoordinatorThriftLimit) { RuntimeState state; const int32_t saved_limit = config::thrift_max_message_size; @@ -113,7 +142,11 @@ TEST(RuntimeStateIcebergCommitDataTest, PeriodicReportOmitsExternalCommitData) { ASSERT_TRUE(state.add_iceberg_commit_datas(iceberg_data).ok()); TMCCommitData mc_data; state.add_mc_commit_datas(mc_data); - ASSERT_TRUE(state.add_connector_commit_data("opaque-fragment").ok()); + TPaimonCommitMessage paimon_message; + paimon_message.__set_payload("paimon-fragment"); + ExternalFileReportReservation paimon_reservation; + ASSERT_TRUE(state.reserve_paimon_commit_messages({paimon_message}, &paimon_reservation).ok()); + state.publish_reserved_paimon_commit_messages({std::move(paimon_message)}, &paimon_reservation); TReportExecStatusParams periodic_params; state.append_external_file_commit_data(&periodic_params, false); @@ -121,16 +154,16 @@ TEST(RuntimeStateIcebergCommitDataTest, PeriodicReportOmitsExternalCommitData) { EXPECT_FALSE(periodic_params.__isset.hive_partition_updates); EXPECT_FALSE(periodic_params.__isset.iceberg_commit_datas); EXPECT_FALSE(periodic_params.__isset.mc_commit_datas); - EXPECT_FALSE(periodic_params.__isset.connector_commit_data); + EXPECT_FALSE(periodic_params.__isset.paimon_commit_messages); TReportExecStatusParams final_params; state.append_external_file_commit_data(&final_params, true); EXPECT_TRUE(final_params.__isset.hive_partition_updates); EXPECT_TRUE(final_params.__isset.iceberg_commit_datas); EXPECT_TRUE(final_params.__isset.mc_commit_datas); - ASSERT_TRUE(final_params.__isset.connector_commit_data); - ASSERT_EQ(1, final_params.connector_commit_data.size()); - EXPECT_EQ("opaque-fragment", final_params.connector_commit_data[0]); + ASSERT_TRUE(final_params.__isset.paimon_commit_messages); + ASSERT_EQ(1, final_params.paimon_commit_messages.size()); + EXPECT_EQ("paimon-fragment", final_params.paimon_commit_messages[0].payload); } TEST(RuntimeStateIcebergCommitDataTest, RetainsFileCleanupUntilReportAcknowledgement) { diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 82064f1ab07e01..d53c3f35a789bc 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2017,8 +2017,11 @@ public class Config extends ConfigBase { public static final int TIMESTAMP_NS_MIN_BE_EXEC_VERSION = 14; // Older backends ignore the optional OpenCSV flag and would silently use different row semantics. public static final int HIVE_OPEN_CSV_MIN_BE_EXEC_VERSION = 15; + // Older backends use branch-4.1 external-sink partition values and routing metadata. + public static final int ALIGNED_EXTERNAL_TABLE_SINK_THRIFT_MIN_BE_EXEC_VERSION = 16; // Older backends do not recognize PAIMON_TABLE_SINK and cannot execute Paimon writes. - public static final int PAIMON_WRITE_MIN_BE_EXEC_VERSION = 16; + public static final int PAIMON_WRITE_MIN_BE_EXEC_VERSION + = ALIGNED_EXTERNAL_TABLE_SINK_THRIFT_MIN_BE_EXEC_VERSION; // Older backends serialize map_agg_v2 keys with the legacy padded frame format. public static final int MAP_AGG_V2_EXACT_FRAME_MIN_BE_EXEC_VERSION = 17; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/ConnectorWritePlanContext.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/ConnectorWritePlanContext.java index b852f12bf33c19..66c0ab5a817756 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/ConnectorWritePlanContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/ConnectorWritePlanContext.java @@ -36,13 +36,14 @@ public class ConnectorWritePlanContext { private final boolean requiresPartitionLocalSort; private final boolean requiresPartitionHashWrite; private final boolean requiresFullSchemaWriteOrder; + private final int beExecVersion; /** Creates an immutable context from one connector-provider resolution. */ public ConnectorWritePlanContext(ConnectorSession session, ConnectorMetadata metadata, ConnectorTableHandle tableHandle, ConnectorWritePlanProvider provider, ConnectorWriteDistribution distribution, boolean requiresParallelWrite, boolean requiresPartitionLocalSort, boolean requiresPartitionHashWrite, - boolean requiresFullSchemaWriteOrder) { + boolean requiresFullSchemaWriteOrder, int beExecVersion) { this.session = session; this.metadata = metadata; this.tableHandle = tableHandle; @@ -52,6 +53,7 @@ public ConnectorWritePlanContext(ConnectorSession session, ConnectorMetadata met this.requiresPartitionLocalSort = requiresPartitionLocalSort; this.requiresPartitionHashWrite = requiresPartitionHashWrite; this.requiresFullSchemaWriteOrder = requiresFullSchemaWriteOrder; + this.beExecVersion = beExecVersion; } public ConnectorSession getSession() { @@ -89,4 +91,8 @@ public boolean requiresPartitionHashWrite() { public boolean requiresFullSchemaWriteOrder() { return requiresFullSchemaWriteOrder; } + + public int getBeExecVersion() { + return beExecVersion; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java index ec9edcbf2a1418..cc95d843239f79 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java @@ -574,7 +574,8 @@ public ConnectorWritePlanContext resolveWritePlanContext() { session, metadata, handle, provider, provider.getWriteDistribution(session, handle), provider.requiresParallelWrite(), provider.requiresPartitionLocalSort(), provider.requiresPartitionHashWrite(), - provider.requiresFullSchemaWriteOrder())); + provider.requiresFullSchemaWriteOrder(), + ConnectContext.get().getStatementContext().getBeExecVersion())); }); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java index 70f998095e63b5..0a80f50d2e610e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java @@ -231,7 +231,7 @@ void checkVariantBackendCompatibilityForCurrentScan(Iterable backends) variantCompatibilityDeferred = true; return; } - checkVariantBackendCompatibility(projectsVariant, backends); + checkVariantBackendCompatibility(projectsVariant, backends, scanContext.getBeExecVersion()); } static boolean projectsComputeVariant(TupleDescriptor tuple) { @@ -260,12 +260,13 @@ private static boolean containsComputeVariant(Type type) { return false; } - static void checkVariantBackendCompatibility(boolean projectsVariant, Iterable backends) + static void checkVariantBackendCompatibility(boolean projectsVariant, Iterable backends, + int beExecVersion) throws UserException { if (!projectsVariant) { return; } - if (Config.be_exec_version < SUPPORT_ICEBERG_VARIANT_EXEC_VERSION) { + if (beExecVersion < SUPPORT_ICEBERG_VARIANT_EXEC_VERSION) { // The query-wide execution version covers every eligible backend, including ordinary // rolling-upgrade nodes that are not marked as cloud smooth-upgrade sources. throw new UserException("Iceberg Variant requires backend execution version " @@ -1126,7 +1127,7 @@ protected TFileAttributes getFileAttributes() throws UserException { if ("true".equals(props.get(ScanNodePropertyKeys.TEXT_HIVE_OPEN_CSV))) { // The optional wire field alone cannot fence old readers during a rolling upgrade. - if (Config.be_exec_version < Config.HIVE_OPEN_CSV_MIN_BE_EXEC_VERSION) { + if (scanContext.getBeExecVersion() < Config.HIVE_OPEN_CSV_MIN_BE_EXEC_VERSION) { throw new UserException("Hive OpenCSVSerde requires backend execution version " + Config.HIVE_OPEN_CSV_MIN_BE_EXEC_VERSION + " or newer during rolling upgrade"); } @@ -1742,7 +1743,7 @@ public List getSplits(int numBackends) throws UserException { if (variantCompatibilityDeferred) { checkVariantBackendCompatibility( plannedScanDecodesVariant(projectsComputeVariant(desc), countPushdown, ranges), - backendPolicy.getBackends()); + backendPolicy.getBackends(), scanContext.getBeExecVersion()); } List splits = new ArrayList<>(ranges.size()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index a0a4bc0bfab3de..05ae6f793d737f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -141,8 +141,16 @@ public NereidsPlanner(StatementContext statementContext) { this.statementContext = statementContext; } + void pinBeExecVersion(org.apache.doris.thrift.TQueryOptions queryOptions) { + if (!statementContext.hasBeExecVersion()) { + statementContext.initializeBeExecVersion(queryOptions.getBeExecVersion()); + } + queryOptions.setBeExecVersion(statementContext.getBeExecVersion()); + } + @Override public void plan(StatementBase queryStmt, org.apache.doris.thrift.TQueryOptions queryOptions) throws UserException { + pinBeExecVersion(queryOptions); this.queryOptions = queryOptions; if (!(queryStmt instanceof LogicalPlanAdapter)) { throw new RuntimeException("Wrong type of queryStmt, expected: "); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 50819630a57505..686a62a51f4fe3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -77,6 +77,7 @@ import org.apache.doris.system.Backend; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.common.base.Strings; import com.google.common.base.Supplier; @@ -132,6 +133,10 @@ public enum TableFrom { } private ConnectContext connectContext; + // Immutable for one statement execution. PREPARE does not initialize this value: each EXECUTE + // receives a fresh StatementContext and NereidsPlanner pins the query-option snapshot before + // analysis starts. + private Integer beExecVersion; // Initialized on first cost calculation so per-query SET_VAR hints have already taken effect. private CostWeight costWeight; private Optional ivmRewriteContext = Optional.empty(); @@ -472,6 +477,26 @@ public StatementContext createNextExecuteContext() { return next; } + /** Pins the backend execution version selected for this statement execution. */ + public void initializeBeExecVersion(int beExecVersion) { + Preconditions.checkState(this.beExecVersion == null || this.beExecVersion == beExecVersion, + "BE execution version is already pinned to %s, cannot change it to %s", + this.beExecVersion, beExecVersion); + this.beExecVersion = beExecVersion; + } + + /** Returns whether this context belongs to an execution whose query options have been pinned. */ + public boolean hasBeExecVersion() { + return beExecVersion != null; + } + + /** Returns the backend execution version pinned for this statement execution. */ + public int getBeExecVersion() { + Preconditions.checkState(beExecVersion != null, + "BE execution version must be initialized at the execution entry point"); + return beExecVersion; + } + public void setNeedLockTables(boolean needLockTables) { this.needLockTables = needLockTables; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 970d640c436a84..633e117c0b9dcf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -565,7 +565,7 @@ public PlanFragment visitPhysicalExternalRowLevelDeleteSink( // BE as the __DORIS_ICEBERG_ROWID_COL__ block column (a real hidden column), and viceberg_delete_sink // resolves it by block-name, not by output-expr name. rootFragment.setSink(buildPluginRowLevelDmlSink(deleteSink, WriteOperation.DELETE, - false, false, deleteSink.getBoundWriteMetadataIdentity())); + false, false, deleteSink.getBoundWriteMetadataIdentity(), context)); return rootFragment; } @@ -603,7 +603,7 @@ public PlanFragment visitPhysicalExternalRowLevelMergeSink( // sink dialect but has no such rule, so it threads false (see PhysicalExternalRowLevelMergeSink). rootFragment.setSink(buildPluginRowLevelDmlSink(mergeSink, WriteOperation.MERGE, mergeSink.isWritesDataFiles(), mergeSink.isRequireMergeCardinalityCheck(), - mergeSink.getBoundWriteMetadataIdentity())); + mergeSink.getBoundWriteMetadataIdentity(), context)); return rootFragment; } @@ -621,7 +621,7 @@ public PlanFragment visitPhysicalExternalRowLevelMergeSink( private PluginDrivenTableSink buildPluginRowLevelDmlSink( PhysicalBaseExternalTableSink sink, WriteOperation writeOperation, boolean writesDataFiles, boolean requireMergeCardinalityCheck, - String boundWriteMetadataIdentity) { + String boundWriteMetadataIdentity, PlanTranslatorContext context) { PluginDrivenExternalTable targetTable = (PluginDrivenExternalTable) sink.getTargetTable(); PluginDrivenExternalCatalog catalog = (PluginDrivenExternalCatalog) targetTable.getCatalog(); @@ -660,7 +660,8 @@ private PluginDrivenTableSink buildPluginRowLevelDmlSink( // connector's TIcebergMergeSink.sort_fields, DELETE is unsorted). return new PluginDrivenTableSink(targetTable, writePlanProvider, connSession, providerTableHandle, connectorColumns, connectorColumns, null, writeOperation, - writesDataFiles, requireMergeCardinalityCheck, boundWriteMetadataIdentity, metadata); + writesDataFiles, requireMergeCardinalityCheck, boundWriteMetadataIdentity, metadata, + context.getBeExecVersion()); } @Override @@ -727,8 +728,8 @@ public PlanFragment visitPhysicalConnectorTableSink( // retain the complete generation captured by BindSink instead of comparing that subset. PluginDrivenTableSink providerSink = new PluginDrivenTableSink(targetTable, writePlanProvider, connSession, providerTableHandle, connectorColumns, - boundTargetColumns, writeSortInfo, writeOperation, false, - boundWriteMetadataIdentity, metadata); + boundTargetColumns, writeSortInfo, writeOperation, true, false, + boundWriteMetadataIdentity, metadata, context.getBeExecVersion()); rootFragment.setSink(providerSink); return rootFragment; } @@ -3805,9 +3806,9 @@ private DataPartition toDataPartition(DistributionSpec distributionSpec/* target partitionExprs.add(context.findSlotRef(partitionExprId)); } } - return new DataPartition(TPartitionType.HIVE_TABLE_SINK_HASH_PARTITIONED, partitionExprs); + return DataPartition.forHiveTableSink(partitionExprs, context.getBeExecVersion()); } else if (distributionSpec instanceof DistributionSpecHiveTableSinkUnPartitioned) { - return new DataPartition(TPartitionType.HIVE_TABLE_SINK_UNPARTITIONED); + return DataPartition.forHiveTableSink(Collections.emptyList(), context.getBeExecVersion()); } else if (distributionSpec instanceof DistributionSpecMerge) { DistributionSpecMerge mergeSpec = (DistributionSpecMerge) distributionSpec; Expr operationExpr = context.findSlotRef(mergeSpec.getOperationExprId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index 476579d2f12b2e..65911a589e9980 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -25,6 +25,7 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; +import org.apache.doris.common.Config; import org.apache.doris.common.IdGenerator; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -171,27 +172,22 @@ public class PlanTranslatorContext { /** PlanTranslatorContext */ public PlanTranslatorContext(CascadesContext ctx) { - this.connectContext = ctx.getConnectContext(); - this.statementContext = ctx.getStatementContext(); - this.scanContext = connectContext == null || connectContext.getSessionVariable() == null - ? ScanContext.EMPTY - : ScanContext.builder() - .clusterName(connectContext.getSessionVariable().resolveCloudClusterName(connectContext)) - .build(); - this.translator = new RuntimeFilterTranslator(ctx.getRuntimeFilterContext()); - this.topnFilterContext = ctx.getTopnFilterContext(); - this.descTable = new DescriptorTable(); + this(ctx, new DescriptorTable()); } /** PlanTranslatorContext */ public PlanTranslatorContext(CascadesContext ctx, DescriptorTable descTable) { this.connectContext = ctx.getConnectContext(); this.statementContext = ctx.getStatementContext(); - this.scanContext = connectContext == null || connectContext.getSessionVariable() == null - ? ScanContext.EMPTY - : ScanContext.builder() - .clusterName(connectContext.getSessionVariable().resolveCloudClusterName(connectContext)) - .build(); + ScanContext.Builder scanContextBuilder = ScanContext.builder(); + if (statementContext.hasBeExecVersion()) { + scanContextBuilder.beExecVersion(statementContext.getBeExecVersion()); + } + if (connectContext != null && connectContext.getSessionVariable() != null) { + scanContextBuilder.clusterName( + connectContext.getSessionVariable().resolveCloudClusterName(connectContext)); + } + this.scanContext = scanContextBuilder.build(); this.translator = new RuntimeFilterTranslator(ctx.getRuntimeFilterContext()); this.topnFilterContext = ctx.getTopnFilterContext(); this.descTable = descTable; @@ -202,14 +198,24 @@ public PlanTranslatorContext(CascadesContext ctx, DescriptorTable descTable) { */ @VisibleForTesting public PlanTranslatorContext() { + this(Config.be_exec_version); + } + + @VisibleForTesting + PlanTranslatorContext(int beExecVersion) { this.connectContext = null; this.statementContext = new StatementContext(); - this.scanContext = ScanContext.EMPTY; + this.statementContext.initializeBeExecVersion(beExecVersion); + this.scanContext = ScanContext.builder().beExecVersion(beExecVersion).build(); this.translator = null; this.topnFilterContext = new TopnFilterContext(); this.descTable = new DescriptorTable(); } + public int getBeExecVersion() { + return statementContext.getBeExecVersion(); + } + /** * remember the unknown-stats column and its scan, used for forbid_unknown_col_stats check */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSink.java index b21e1e5d984642..749c1bf96b3250 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSink.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.plans.physical; import org.apache.doris.catalog.Column; -import org.apache.doris.common.Config; import org.apache.doris.connector.spi.write.ConnectorWriteDistribution; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; @@ -508,7 +507,7 @@ private PhysicalProperties toPhysicalProperties(ConnectorWriteDistribution distr } private void requireExternalWriterRoutingSupport() { - Preconditions.checkState(Config.be_exec_version + Preconditions.checkState(getWritePlanContext().getBeExecVersion() >= DistributionSpecExternalTableSinkHashPartitioned.MIN_BE_EXEC_VERSION, "External table sink distribution requires BE execution version %s or newer", DistributionSpecExternalTableSinkHashPartitioned.MIN_BE_EXEC_VERSION); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java index 9a2385b8a3350f..84af2b522ea688 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java @@ -24,8 +24,10 @@ import org.apache.doris.analysis.ExprToSqlVisitor; import org.apache.doris.analysis.ExprToThriftVisitor; import org.apache.doris.analysis.ToSqlParams; +import org.apache.doris.common.Config; import org.apache.doris.thrift.TDataPartition; import org.apache.doris.thrift.TExplainLevel; +import org.apache.doris.thrift.TExternalTableSinkHashAlgorithm; import org.apache.doris.thrift.TExternalTableSinkHashPartitionInfo; import org.apache.doris.thrift.TExternalTableSinkWriterAssignment; import org.apache.doris.thrift.TIcebergPartitionField; @@ -62,6 +64,7 @@ public class DataPartition { private String externalPartitionFunction; private ImmutableMap externalPartitionFunctionOptions = ImmutableMap.of(); private TExternalTableSinkWriterAssignment externalWriterAssignment; + private TExternalTableSinkHashAlgorithm externalPartitionAlgorithm; public DataPartition(TPartitionType type, List exprs) { Preconditions.checkNotNull(exprs); @@ -78,11 +81,19 @@ public DataPartition(TPartitionType type, List exprs) { public DataPartition(TPartitionType type, List exprs, String partitionFunction, Map partitionFunctionOptions, TExternalTableSinkWriterAssignment writerAssignment) { + this(type, exprs, partitionFunction, partitionFunctionOptions, writerAssignment, null); + } + + private DataPartition(TPartitionType type, List exprs, String partitionFunction, + Map partitionFunctionOptions, + TExternalTableSinkWriterAssignment writerAssignment, + TExternalTableSinkHashAlgorithm partitionAlgorithm) { this(type, exprs); Preconditions.checkState(type == TPartitionType.EXTERNAL_TABLE_SINK_HASH_PARTITIONED); this.externalPartitionFunction = Preconditions.checkNotNull(partitionFunction); this.externalPartitionFunctionOptions = ImmutableMap.copyOf(partitionFunctionOptions); this.externalWriterAssignment = Preconditions.checkNotNull(writerAssignment); + this.externalPartitionAlgorithm = partitionAlgorithm; } public DataPartition(TPartitionType type) { @@ -134,11 +145,32 @@ public TDataPartition toThrift() { info.setPartitionFunction(externalPartitionFunction); info.setPartitionFunctionOptions(externalPartitionFunctionOptions); info.setWriterAssignment(externalWriterAssignment); + if (externalPartitionAlgorithm != null) { + info.setAlgorithm(externalPartitionAlgorithm); + } result.setExternalTableSinkHashPartitionInfo(info); } return result; } + /** Builds the complete Hive table-sink wire dialect for one statement execution version. */ + public static DataPartition forHiveTableSink(List partitionExprs, int beExecVersion) { + if (partitionExprs.isEmpty()) { + TPartitionType type = beExecVersion + < Config.ALIGNED_EXTERNAL_TABLE_SINK_THRIFT_MIN_BE_EXEC_VERSION + ? TPartitionType.EXTERNAL_TABLE_SINK_UNPARTITIONED + : TPartitionType.HIVE_TABLE_SINK_UNPARTITIONED; + return new DataPartition(type); + } + if (beExecVersion < Config.ALIGNED_EXTERNAL_TABLE_SINK_THRIFT_MIN_BE_EXEC_VERSION) { + return new DataPartition(TPartitionType.EXTERNAL_TABLE_SINK_HASH_PARTITIONED, + partitionExprs, "direct_hash", ImmutableMap.of(), + TExternalTableSinkWriterAssignment.SKEWED, + TExternalTableSinkHashAlgorithm.DIRECT_HASH); + } + return new DataPartition(TPartitionType.HIVE_TABLE_SINK_HASH_PARTITIONED, partitionExprs); + } + public String getExplainString(TExplainLevel explainLevel) { StringBuilder str = new StringBuilder(); str.append(type.toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PluginDrivenTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PluginDrivenTableSink.java index 08098fe31407a7..fbbc7bfcaa3a7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PluginDrivenTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PluginDrivenTableSink.java @@ -19,7 +19,6 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; import org.apache.doris.connector.spi.ConnectorColumn; import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorSession; @@ -88,127 +87,10 @@ public class PluginDrivenTableSink extends BaseExternalTableDataSink { // Carried from PhysicalExternalRowLevelMergeSink onto the write handle so the connector can stamp the // enforcement flag onto its BE sink; false for UPDATE and for every non-row-level write. private final boolean requireMergeCardinalityCheck; + private final int beExecVersion; /** - * Plan-provider mode (W5): the connector supplies a {@link ConnectorWritePlanProvider} - * and builds its own opaque {@link TDataSink} via - * {@link ConnectorWritePlanProvider#planWrite}. - */ - public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, - ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, - ConnectorTableHandle tableHandle, List connectorColumns) { - this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, null); - } - - /** - * Plan-provider mode with an engine-built write {@link TSortInfo} threaded to the connector's write - * handle (for a connector that declares write-sort columns). - */ - public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, - ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, - ConnectorTableHandle tableHandle, List connectorColumns, - TSortInfo writeSortInfo) { - this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, - writeSortInfo, WriteOperation.INSERT); - } - - /** - * Plan-provider mode with the DML {@link WriteOperation} threaded to the connector's write handle, so - * the connector's {@code planWrite} dispatches to the matching BE sink dialect. The two shorter ctors - * default this to {@link WriteOperation#INSERT} (the byte-identical plain-INSERT path); the row-level - * DML translator arms use this ctor with {@code DELETE} / {@code UPDATE} / {@code MERGE}. - */ - public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, - ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, - ConnectorTableHandle tableHandle, List connectorColumns, - TSortInfo writeSortInfo, WriteOperation writeOperation) { - this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, - writeSortInfo, writeOperation, false, null); - } - - /** - * Plan-provider mode with the SQL MERGE cardinality requirement threaded to the connector's write - * handle. Only the MERGE translator arm passes {@code true}; every other ctor defaults it to - * {@code false}, so UPDATE / DELETE / INSERT keep their byte-identical sink. - */ - public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, - ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, - ConnectorTableHandle tableHandle, List connectorColumns, - TSortInfo writeSortInfo, WriteOperation writeOperation, boolean requireMergeCardinalityCheck) { - this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, - writeSortInfo, writeOperation, requireMergeCardinalityCheck, null); - } - - /** - * Plan-provider mode with connector metadata used to resolve the exact branch-aware MVCC pin at bind time. - */ - public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, - ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, - ConnectorTableHandle tableHandle, List connectorColumns, - TSortInfo writeSortInfo, WriteOperation writeOperation, boolean requireMergeCardinalityCheck, - ConnectorMetadata connectorMetadata) { - this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, - connectorColumns, writeSortInfo, writeOperation, true, - requireMergeCardinalityCheck, null, connectorMetadata); - } - - /** - * Plan-provider mode with explicit data-file and merge-cardinality requirements. - */ - public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, - ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, - ConnectorTableHandle tableHandle, List connectorColumns, - TSortInfo writeSortInfo, WriteOperation writeOperation, boolean writesDataFiles, - boolean requireMergeCardinalityCheck) { - this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, - connectorColumns, writeSortInfo, writeOperation, writesDataFiles, - requireMergeCardinalityCheck, null, null); - } - - /** - * Plan-provider mode with the write subset and complete bind-time target schema carried separately. - */ - public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, - ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, - ConnectorTableHandle tableHandle, List connectorColumns, - List boundTargetColumns, TSortInfo writeSortInfo, - WriteOperation writeOperation, boolean requireMergeCardinalityCheck) { - this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, - boundTargetColumns, writeSortInfo, writeOperation, true, - requireMergeCardinalityCheck, null, null); - } - - /** - * Plan-provider mode with the connector metadata generation that shaped this physical write. - */ - public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, - ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, - ConnectorTableHandle tableHandle, List connectorColumns, - List boundTargetColumns, TSortInfo writeSortInfo, - WriteOperation writeOperation, boolean requireMergeCardinalityCheck, - String boundWriteMetadataIdentity) { - this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, - boundTargetColumns, writeSortInfo, writeOperation, true, - requireMergeCardinalityCheck, boundWriteMetadataIdentity, null); - } - - /** - * Complete plan-provider mode. The schema generation and branch-aware snapshot are separate invariants: - * the former must stay at bind time, while the latter is resolved for the exact write branch. - */ - public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, - ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, - ConnectorTableHandle tableHandle, List connectorColumns, - List boundTargetColumns, TSortInfo writeSortInfo, - WriteOperation writeOperation, boolean requireMergeCardinalityCheck, - String boundWriteMetadataIdentity, ConnectorMetadata connectorMetadata) { - this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, - boundTargetColumns, writeSortInfo, writeOperation, true, - requireMergeCardinalityCheck, boundWriteMetadataIdentity, connectorMetadata); - } - - /** - * Plan-provider mode with explicit schema, data-file, metadata-generation, and MVCC settings. + * Complete plan-provider mode with the statement-scoped backend execution version. */ public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, @@ -216,7 +98,7 @@ public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, List boundTargetColumns, TSortInfo writeSortInfo, WriteOperation writeOperation, boolean writesDataFiles, boolean requireMergeCardinalityCheck, String boundWriteMetadataIdentity, - ConnectorMetadata connectorMetadata) { + ConnectorMetadata connectorMetadata, int beExecVersion) { super(); this.targetTable = targetTable; this.writePlanProvider = writePlanProvider; @@ -232,6 +114,7 @@ public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, this.writeOperation = writeOperation == null ? WriteOperation.INSERT : writeOperation; this.writesDataFiles = writesDataFiles; this.requireMergeCardinalityCheck = requireMergeCardinalityCheck; + this.beExecVersion = beExecVersion; } /** @@ -265,7 +148,7 @@ public String getExplainString(String prefix, TExplainLevel explainLevel) { ConnectorWriteHandle handle = new PluginDrivenWriteHandle( tableHandle, connectorColumns, boundTargetColumns, false, Collections.emptyMap(), Collections.emptySet(), null, null, Optional.empty(), writeOperation, - writesDataFiles, requireMergeCardinalityCheck); + writesDataFiles, requireMergeCardinalityCheck, beExecVersion); writePlanProvider.appendExplainInfo(sb, prefix, connectorSession, handle); return sb.toString(); } @@ -283,7 +166,7 @@ public void bindDataSink(Optional insertCtx) throws AnalysisException { if (writeOperation == WriteOperation.MERGE && !writesDataFiles && containsVariant(boundTargetColumns) - && Config.be_exec_version < SUPPORT_ICEBERG_VARIANT_EXEC_VERSION) { + && beExecVersion < SUPPORT_ICEBERG_VARIANT_EXEC_VERSION) { // Older BEs ignore writes_data_files and instantiate the omitted data writer, so reject // only when the schema would make that legacy writer fail to parse Variant metadata. throw new AnalysisException("Delete-only Iceberg MERGE with Variant is unavailable " @@ -314,7 +197,7 @@ && containsVariant(boundTargetColumns) boundTableHandle, connectorColumns, boundTargetColumns, overwrite, writeContext, staticPartitionNullKeys, writeSortInfo, boundWriteMetadataIdentity, branchName, writeOperation, writesDataFiles, - requireMergeCardinalityCheck); + requireMergeCardinalityCheck, beExecVersion); ConnectorSinkPlan sinkPlan = writePlanProvider.planWrite(connectorSession, handle); this.tDataSink = sinkPlan.getDataSink(); } @@ -353,13 +236,15 @@ private static final class PluginDrivenWriteHandle implements ConnectorWriteHand private final WriteOperation writeOperation; private final boolean writesDataFiles; private final boolean requireMergeCardinalityCheck; + private final int beExecVersion; private PluginDrivenWriteHandle(ConnectorTableHandle tableHandle, List columns, List boundTargetColumns, boolean overwrite, Map writeContext, Set staticPartitionNullKeys, TSortInfo sortInfo, String boundWriteMetadataIdentity, Optional branchName, WriteOperation writeOperation, - boolean writesDataFiles, boolean requireMergeCardinalityCheck) { + boolean writesDataFiles, boolean requireMergeCardinalityCheck, + int beExecVersion) { this.tableHandle = tableHandle; this.columns = columns; this.boundTargetColumns = boundTargetColumns; @@ -372,6 +257,7 @@ private PluginDrivenWriteHandle(ConnectorTableHandle tableHandle, List @@ -24,12 +26,14 @@ * we can evolve fields incrementally in future. */ public final class ScanContext { - public static final ScanContext EMPTY = new ScanContext(""); + public static final ScanContext EMPTY = new ScanContext("", null); private final String clusterName; + private final Integer beExecVersion; - private ScanContext(String clusterName) { + private ScanContext(String clusterName, Integer beExecVersion) { this.clusterName = clusterName == null ? "" : clusterName; + this.beExecVersion = beExecVersion; } public static Builder builder() { @@ -40,19 +44,28 @@ public String getClusterName() { return clusterName; } + public int getBeExecVersion() { + Preconditions.checkState(beExecVersion != null, + "BE execution version is unavailable outside an execution planning context"); + return beExecVersion; + } + public static final class Builder { private String clusterName = ""; + private Integer beExecVersion; public Builder clusterName(String clusterName) { this.clusterName = clusterName; return this; } + public Builder beExecVersion(int beExecVersion) { + this.beExecVersion = beExecVersion; + return this; + } + public ScanContext build() { - if (clusterName == null || clusterName.isEmpty()) { - return ScanContext.EMPTY; - } - return new ScanContext(clusterName); + return new ScanContext(clusterName, beExecVersion); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index a96dc7e3b32e50..99521b61ad0921 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -40,7 +40,6 @@ import org.apache.doris.datasource.scan.FileQueryScanNode; import org.apache.doris.load.loadv2.LoadJob; import org.apache.doris.metric.MetricRepo; -import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.stats.StatsErrorEstimator; import org.apache.doris.nereids.trees.plans.distribute.DistributedPlan; @@ -86,7 +85,6 @@ import org.apache.doris.resource.workloadgroup.WorkloadGroup; import org.apache.doris.rpc.BackendServiceProxy; import org.apache.doris.rpc.RpcException; -import org.apache.doris.service.ExecuteEnv; import org.apache.doris.service.FrontendOptions; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; @@ -116,7 +114,6 @@ import org.apache.doris.thrift.TQueryType; import org.apache.doris.thrift.TReportExecStatusParams; import org.apache.doris.thrift.TResourceInfo; -import org.apache.doris.thrift.TResourceLimit; import org.apache.doris.thrift.TRuntimeFilterParams; import org.apache.doris.thrift.TRuntimeFilterTargetParamsV2; import org.apache.doris.thrift.TScanRange; @@ -357,7 +354,10 @@ public Coordinator(ConnectContext context, Planner planner) { this.returnedAllResults = false; this.enableShareHashTableForBroadcastJoin = context.getSessionVariable().enableShareHashTableForBroadcastJoin; - initQueryOptions(context); + this.queryOptions = CoordinatorContext.initQueryOptions( + context, planner.getQueryOptions().getBeExecVersion()); + // Old Coordinator never plans local exchange in FE. Force BE to plan its own. + this.queryOptions.setEnableLocalShufflePlanner(false); useNereids = planner instanceof NereidsPlanner; if (!useNereids) { // Enable local shuffle on pipelineX engine only if Nereids planner is applied. @@ -365,8 +365,6 @@ public Coordinator(ConnectContext context, Planner planner) { } else { distributedPlans = ((NereidsPlanner) planner).getDistributedPlans(); } - setFromUserProperty(context); - CoordinatorContext.refreshQueryGlobals(this.queryGlobals, context); this.queryGlobals.setLoadZeroTolerance(false); this.assignedRuntimeFilters = planner.getRuntimeFilters(); @@ -404,38 +402,6 @@ public Coordinator(Long jobId, TUniqueId queryId, DescriptorTable descTable, Lis this.executionProfile = new ExecutionProfile(queryId, fragmentIds); } - private void setFromUserProperty(ConnectContext connectContext) { - String qualifiedUser = connectContext.getQualifiedUser(); - // set cpu resource limitch - int cpuLimit = Env.getCurrentEnv().getAuth().getCpuResourceLimit(qualifiedUser); - if (cpuLimit > 0) { - // overwrite the cpu resource limit from session variable; - TResourceLimit resourceLimit = new TResourceLimit(); - resourceLimit.setCpuLimit(cpuLimit); - this.queryOptions.setResourceLimit(resourceLimit); - } - // set exec mem limit - long memLimit = connectContext.getMaxExecMemByte(); - if (memLimit > 0) { - // overwrite the exec_mem_limit from session variable; - this.queryOptions.setMemLimit(memLimit); - } - } - - private void initQueryOptions(ConnectContext context) { - this.queryOptions = context.getSessionVariable().toThrift(); - this.queryOptions.setQueryTimeout(context.getExecTimeoutS()); - this.queryOptions.setExecutionTimeout(context.getExecTimeoutS()); - if (this.queryOptions.getExecutionTimeout() < 1) { - LOG.info("try set timeout less than 1", new RuntimeException("")); - } - this.queryOptions.setFeProcessUuid(ExecuteEnv.getInstance().getProcessUUID()); - this.queryOptions.setMysqlRowBinaryFormat( - context.getCommand() == MysqlCommand.COM_STMT_EXECUTE); - // Old Coordinator never plans local exchange in FE. Force BE to plan its own. - this.queryOptions.setEnableLocalShufflePlanner(false); - } - public ConnectContext getConnectContext() { return context; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java index 9e21d760f7c913..9ddd2a1aa7545e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java @@ -276,7 +276,8 @@ public QueryProcessor asQueryProcessor() { public static CoordinatorContext buildForSql(NereidsPlanner planner, NereidsCoordinator coordinator) { ConnectContext connectContext = planner.getCascadesContext().getConnectContext(); - TQueryOptions queryOptions = initQueryOptions(connectContext); + TQueryOptions queryOptions = initQueryOptions( + connectContext, planner.getQueryOptions().getBeExecVersion()); TQueryGlobals queryGlobals = createQueryGlobals(connectContext); TDescriptorTable descriptorTable = DescriptorToThriftConverter.toThrift(planner.getDescTable()); @@ -327,9 +328,9 @@ public static CoordinatorContext buildForLoad( executionProfile); } - private static TQueryOptions initQueryOptions(ConnectContext context) { + static TQueryOptions initQueryOptions(ConnectContext context, int beExecVersion) { TQueryOptions queryOptions = context.getSessionVariable().toThrift(); - queryOptions.setBeExecVersion(Config.be_exec_version); + queryOptions.setBeExecVersion(beExecVersion); queryOptions.setQueryTimeout(context.getExecTimeoutS()); queryOptions.setExecutionTimeout(context.getExecTimeoutS()); if (queryOptions.getExecutionTimeout() < 1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PluginDrivenQueryTableValueFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PluginDrivenQueryTableValueFunction.java index 19089bd0526fb3..628fc99f9a500c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PluginDrivenQueryTableValueFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PluginDrivenQueryTableValueFunction.java @@ -32,6 +32,7 @@ import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.planner.ScanNode; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.logging.log4j.LogManager; @@ -73,7 +74,9 @@ public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable ConnectorSession session = pluginCatalog.buildConnectorSession(); PassthroughQueryTableHandle queryHandle = new PassthroughQueryTableHandle(query); return new PluginDrivenScanNode(id, desc, false, sv, - ScanContext.builder().clusterName(sv.resolveCloudClusterName()).build(), + ScanContext.builder().clusterName(sv.resolveCloudClusterName()) + .beExecVersion(ConnectContext.get().getStatementContext().getBeExecVersion()) + .build(), pluginCatalog.getConnector(), session, queryHandle); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/CommitDataSerializer.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/CommitDataSerializer.java index a9738ec030a720..c477bd46b4db57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/CommitDataSerializer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/CommitDataSerializer.java @@ -24,7 +24,6 @@ import org.apache.thrift.TSerializer; import org.apache.thrift.protocol.TBinaryProtocol; -import java.nio.ByteBuffer; import java.util.List; import java.util.stream.Collectors; @@ -35,7 +34,7 @@ * *

This is the single place the FE-side serialization protocol is defined. It * MUST match the deserialization protocol used by the write transactions' - * {@code addCommitData} overrides (maxcompute / hive / iceberg); the + * {@code addCommitData} overrides (maxcompute / hive / iceberg / paimon); the * {@code CommitDataSerializerTest} golden tests pin that agreement.

*/ public final class CommitDataSerializer { @@ -46,7 +45,7 @@ private CommitDataSerializer() { /** Returns whether a fragment report carries any external connector commit data. */ public static boolean hasCommitData(TReportExecStatusParams params) { return params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() - || params.isSetMcCommitDatas() || params.isSetConnectorCommitData(); + || params.isSetMcCommitDatas() || params.isSetPaimonCommitMessages(); } /** Delivers every commit-data representation carried by one fragment report. */ @@ -60,8 +59,8 @@ public static void feed(Transaction txn, TReportExecStatusParams params) { if (params.isSetMcCommitDatas()) { feed(txn, params.getMcCommitDatas()); } - if (params.isSetConnectorCommitData()) { - feedRaw(txn, params.getConnectorCommitData()); + if (params.isSetPaimonCommitMessages()) { + feed(txn, params.getPaimonCommitMessages()); } } @@ -92,20 +91,6 @@ public static void feed(Transaction txn, List> fragments) } } - /** - * Delivers opaque commit fragments without interpreting connector-owned bytes in FE core. - * Thrift exposes binary values as {@link ByteBuffer}; copy each remaining slice before - * passing it to a transaction, which may keep the byte array after the RPC is released. - */ - public static void feedRaw(Transaction txn, List fragments) { - for (ByteBuffer fragment : fragments) { - ByteBuffer source = fragment.duplicate(); - byte[] bytes = new byte[source.remaining()]; - source.get(bytes); - txn.addCommitData(bytes); - } - } - private static final class CommitDataSerializationException extends RuntimeException { private CommitDataSerializationException(TException cause) { super(cause); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCompatibilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCompatibilityTest.java index 7b03c7221b7218..839013a271f5e4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCompatibilityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCompatibilityTest.java @@ -29,7 +29,6 @@ import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; -import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.connector.spi.Connector; @@ -43,6 +42,7 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; +import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.system.Backend; @@ -117,7 +117,7 @@ public void computeVariantRejectsSmoothUpgradeSourceBackend() { UserException exception = Assertions.assertThrows(UserException.class, () -> PluginDrivenScanNode.checkVariantBackendCompatibility( - true, Collections.singletonList(backend))); + true, Collections.singletonList(backend), VARIANT_EXEC_VERSION)); Assertions.assertTrue(exception.getMessage().contains("backend 7")); } @@ -127,23 +127,17 @@ public void compatibilityCheckIgnoresScansWithoutComputeVariant() throws UserExc backend.setSmoothUpgradeSrc(true); PluginDrivenScanNode.checkVariantBackendCompatibility( - false, Collections.singletonList(backend)); + false, Collections.singletonList(backend), VARIANT_EXEC_VERSION); } @Test public void computeVariantRejectsOldQueryWideExecutionVersion() { - int original = Config.be_exec_version; - try { - Config.be_exec_version = VARIANT_EXEC_VERSION - 1; - Backend backend = new Backend(8L, "127.0.0.1", 9050); + Backend backend = new Backend(8L, "127.0.0.1", 9050); - UserException exception = Assertions.assertThrows(UserException.class, - () -> PluginDrivenScanNode.checkVariantBackendCompatibility( - true, Collections.singletonList(backend))); - Assertions.assertTrue(exception.getMessage().contains("execution version")); - } finally { - Config.be_exec_version = original; - } + UserException exception = Assertions.assertThrows(UserException.class, + () -> PluginDrivenScanNode.checkVariantBackendCompatibility( + true, Collections.singletonList(backend), VARIANT_EXEC_VERSION - 1)); + Assertions.assertTrue(exception.getMessage().contains("execution version")); } @Test @@ -203,22 +197,16 @@ public Map getProperties() { }; List backends = Collections.singletonList( new Backend(9L, "127.0.0.1", 9050)); - int original = Config.be_exec_version; - try { - Config.be_exec_version = VARIANT_EXEC_VERSION - 1; - PluginDrivenScanNode.checkVariantBackendCompatibility( - PluginDrivenScanNode.plannedScanDecodesVariant( - true, true, Collections.singletonList(countRange)), - backends); - - Assertions.assertThrows(UserException.class, - () -> PluginDrivenScanNode.checkVariantBackendCompatibility( - PluginDrivenScanNode.plannedScanDecodesVariant( - true, true, Arrays.asList(countRange, dataRange)), - backends)); - } finally { - Config.be_exec_version = original; - } + PluginDrivenScanNode.checkVariantBackendCompatibility( + PluginDrivenScanNode.plannedScanDecodesVariant( + true, true, Collections.singletonList(countRange)), + backends, VARIANT_EXEC_VERSION - 1); + + Assertions.assertThrows(UserException.class, + () -> PluginDrivenScanNode.checkVariantBackendCompatibility( + PluginDrivenScanNode.plannedScanDecodesVariant( + true, true, Arrays.asList(countRange, dataRange)), + backends, VARIANT_EXEC_VERSION - 1)); } @Test @@ -267,6 +255,8 @@ private static PluginDrivenScanNode metadataCountVariantNode(ConnectorScanPlanPr Deencapsulation.setField(node, "connector", connector); Deencapsulation.setField(node, "connectorSession", Mockito.mock(ConnectorSession.class)); Deencapsulation.setField(node, "currentHandle", new ConnectorTableHandle() { }); + Deencapsulation.setField(node, "scanContext", + ScanContext.builder().beExecVersion(VARIANT_EXEC_VERSION).build()); Deencapsulation.setField(node, "conjuncts", new ArrayList<>()); Deencapsulation.setField(node, "pushDownAggNoGroupingOp", TPushAggOp.COUNT); Deencapsulation.setField(node, "pushDownCountSlotIds", Collections.emptyList()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCsvPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCsvPropertiesTest.java index 503cb68df4193c..a3703964adb09f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCsvPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCsvPropertiesTest.java @@ -23,6 +23,7 @@ import org.apache.doris.connector.hive.HiveTextProperties; import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.connector.spi.scan.ScanNodePropertyKeys; +import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TFileAttributes; @@ -144,6 +145,24 @@ void testOpenCsvAcceptsSupportingExecutionVersion() throws Exception { } } + @Test + void testOpenCsvUsesStatementExecutionVersionSnapshot() { + int original = Config.be_exec_version; + try { + Config.be_exec_version = Config.HIVE_OPEN_CSV_MIN_BE_EXEC_VERSION - 1; + ScanContext scanContext = ScanContext.builder() + .beExecVersion(Config.be_exec_version).build(); + Config.be_exec_version = Config.HIVE_OPEN_CSV_MIN_BE_EXEC_VERSION; + + UserException error = Assertions.assertThrows(UserException.class, + () -> propertiesToWire( + HiveTextProperties.extract(CSV_SERDE, Map.of(), Map.of()), scanContext)); + Assertions.assertTrue(error.getMessage().contains("OpenCSV")); + } finally { + Config.be_exec_version = original; + } + } + @Test void testAbsentSemanticFlagRetainsLegacyWireContract() throws Exception { int original = Config.be_exec_version; @@ -192,9 +211,16 @@ private static TFileAttributes toWire(Map serdeProperties, } private static TFileAttributes propertiesToWire(Map properties) throws Exception { + return propertiesToWire(properties, + ScanContext.builder().beExecVersion(Config.be_exec_version).build()); + } + + private static TFileAttributes propertiesToWire(Map properties, + ScanContext scanContext) throws Exception { PluginDrivenScanNode node = Mockito.mock(PluginDrivenScanNode.class, Mockito.CALLS_REAL_METHODS); Deencapsulation.setField(node, "scanNodeProperties", properties); Deencapsulation.setField(node, "sessionVariable", new SessionVariable()); + Deencapsulation.setField(node, "scanContext", scanContext); // Exercise the real scan-node consumer and wire encoding: map assertions cannot detect byte truncation. TFileAttributes attributes = node.getFileAttributes(); TFileAttributes decoded = new TFileAttributes(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/NereidsPlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/NereidsPlannerTest.java index b57a30a2d1578f..1198808a9e5628 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/NereidsPlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/NereidsPlannerTest.java @@ -23,12 +23,28 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.thrift.TQueryOptions; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; public class NereidsPlannerTest { + @Test + public void testReplanningReusesPinnedBeExecVersion() { + StatementContext statementContext = new StatementContext(); + NereidsPlanner planner = new NereidsPlanner(statementContext); + TQueryOptions firstAttempt = new TQueryOptions().setBeExecVersion(13); + TQueryOptions retryAttempt = new TQueryOptions().setBeExecVersion(16); + + planner.pinBeExecVersion(firstAttempt); + planner.pinBeExecVersion(retryAttempt); + + Assertions.assertEquals(13, firstAttempt.getBeExecVersion()); + Assertions.assertEquals(13, retryAttempt.getBeExecVersion()); + } + @Test public void testCollectAndLockTableRecordsPreloadTimeWhenExecuted() { SummaryProfile summaryProfile = Mockito.mock(SummaryProfile.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java index 61a9653ca34839..38dd1969f4b4e5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java @@ -63,6 +63,21 @@ public void testRowBinlogReferenceTsoIsStatementScoped() { statementContext::getRowBinlogReferenceTso); } + @Test + public void testExecutionVersionIsFreshForEachPreparedExecution() { + StatementContext firstExecution = new StatementContext(); + firstExecution.initializeBeExecVersion(13); + + StatementContext secondExecution = firstExecution.createNextExecuteContext(); + org.junit.jupiter.api.Assertions.assertFalse(secondExecution.hasBeExecVersion()); + secondExecution.initializeBeExecVersion(16); + + org.junit.jupiter.api.Assertions.assertEquals(13, firstExecution.getBeExecVersion()); + org.junit.jupiter.api.Assertions.assertEquals(16, secondExecution.getBeExecVersion()); + org.junit.jupiter.api.Assertions.assertThrows(IllegalStateException.class, + () -> secondExecution.initializeBeExecVersion(15)); + } + @Test public void testSkipPreloadWhenSessionVariableDisabled() { ConnectContext connectContext = Mockito.mock(ConnectContext.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorAdmissionGateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorAdmissionGateTest.java index 72bd39a5d9682e..ff82ee634a99e8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorAdmissionGateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorAdmissionGateTest.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorColumn; @@ -288,7 +289,7 @@ private static void attachWritePlanContext(PhysicalConnectorTableSink sink ConnectorWritePlanProvider provider = connector.getWritePlanProvider(handle); ConnectorWritePlanContext writePlanContext = new ConnectorWritePlanContext( session, metadata, handle, provider, null, false, false, false, - table.requiresFullSchemaWriteOrder()); + table.requiresFullSchemaWriteOrder(), Config.max_be_exec_version); Mockito.doReturn(writePlanContext).when(sink).getWritePlanContext(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorHiveWriteCompatibilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorHiveWriteCompatibilityTest.java new file mode 100644 index 00000000000000..3d013d3396e267 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorHiveWriteCompatibilityTest.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.glue.translator; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.common.Config; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.planner.DataPartition; +import org.apache.doris.thrift.TDataPartition; +import org.apache.doris.thrift.TExternalTableSinkHashAlgorithm; +import org.apache.doris.thrift.TExternalTableSinkWriterAssignment; +import org.apache.doris.thrift.TPartitionType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; + +class PhysicalPlanTranslatorHiveWriteCompatibilityTest { + + @Test + void usesBranch41ExternalRoutingUntilAllBackendsSupportAlignedValues() { + List partitionExprs = Collections.singletonList(new IntLiteral(1)); + + DataPartition partition = DataPartition.forHiveTableSink( + partitionExprs, Config.ALIGNED_EXTERNAL_TABLE_SINK_THRIFT_MIN_BE_EXEC_VERSION - 1); + TDataPartition thrift = partition.toThrift(); + + Assertions.assertEquals(TPartitionType.EXTERNAL_TABLE_SINK_HASH_PARTITIONED, thrift.type); + Assertions.assertEquals(TExternalTableSinkHashAlgorithm.DIRECT_HASH, + thrift.external_table_sink_hash_partition_info.algorithm); + Assertions.assertEquals(TExternalTableSinkWriterAssignment.SKEWED, + thrift.external_table_sink_hash_partition_info.writer_assignment); + Assertions.assertEquals("direct_hash", + thrift.external_table_sink_hash_partition_info.partition_function); + } + + @Test + void usesAlignedHiveRoutingAfterRollingUpgrade() { + List partitionExprs = Collections.singletonList(new IntLiteral(1)); + + DataPartition partition = DataPartition.forHiveTableSink( + partitionExprs, Config.ALIGNED_EXTERNAL_TABLE_SINK_THRIFT_MIN_BE_EXEC_VERSION); + + Assertions.assertEquals(TPartitionType.HIVE_TABLE_SINK_HASH_PARTITIONED, + partition.toThrift().type); + } + + @Test + void preservesUnpartitionedHiveRoutingAcrossRollingUpgrade() { + DataPartition legacy = DataPartition.forHiveTableSink( + Collections.emptyList(), + Config.ALIGNED_EXTERNAL_TABLE_SINK_THRIFT_MIN_BE_EXEC_VERSION - 1); + DataPartition aligned = DataPartition.forHiveTableSink( + Collections.emptyList(), + Config.ALIGNED_EXTERNAL_TABLE_SINK_THRIFT_MIN_BE_EXEC_VERSION); + + Assertions.assertEquals(TPartitionType.EXTERNAL_TABLE_SINK_UNPARTITIONED, + legacy.toThrift().type); + Assertions.assertEquals(TPartitionType.HIVE_TABLE_SINK_UNPARTITIONED, + aligned.toThrift().type); + } + + @Test + void usesStatementVersionSnapshotWhenGlobalVersionChanges() { + int originalBeExecVersion = Config.be_exec_version; + try { + CascadesContext cascadesContext = CascadesContext.initTempContext(); + cascadesContext.getStatementContext().initializeBeExecVersion( + Config.ALIGNED_EXTERNAL_TABLE_SINK_THRIFT_MIN_BE_EXEC_VERSION); + PlanTranslatorContext context = new PlanTranslatorContext(cascadesContext); + + Config.be_exec_version = Config.ALIGNED_EXTERNAL_TABLE_SINK_THRIFT_MIN_BE_EXEC_VERSION - 1; + DataPartition partition = DataPartition.forHiveTableSink( + Collections.emptyList(), context.getBeExecVersion()); + + Assertions.assertEquals(TPartitionType.HIVE_TABLE_SINK_UNPARTITIONED, + partition.toThrift().type); + } finally { + Config.be_exec_version = originalBeExecVersion; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertExecutorTest.java index 75fd22c20ce220..1c1f98a73a7d6d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertExecutorTest.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands.insert; import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.connector.ConnectorSessionBuilder; @@ -27,6 +28,7 @@ import org.apache.doris.connector.spi.handle.ConnectorTransaction; import org.apache.doris.connector.spi.handle.ConnectorWriteHandle; import org.apache.doris.connector.spi.handle.NoOpConnectorTransaction; +import org.apache.doris.connector.spi.handle.WriteOperation; import org.apache.doris.connector.spi.write.ConnectorSinkPlan; import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider; import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; @@ -113,7 +115,9 @@ public void finalizeSinkBindsTransactionOntoSinkSessionBeforePlanWrite() { ConnectorSession sinkSession = ConnectorSessionBuilder.create().build(); RecordingWritePlanProvider provider = new RecordingWritePlanProvider(); PluginDrivenTableSink sink = new PluginDrivenTableSink( - null, provider, sinkSession, new ConnectorTableHandle() { }, Collections.emptyList()); + null, provider, sinkSession, new ConnectorTableHandle() { }, Collections.emptyList(), + Collections.emptyList(), null, WriteOperation.INSERT, true, false, + null, null, Config.max_be_exec_version); exec.finalizeSink(null, sink, null); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSinkTest.java index 7364ff17302dab..4c151cc8e7fe69 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSinkTest.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.connector.spi.write.ConnectorWriteDistribution; import org.apache.doris.datasource.ExternalDatabase; @@ -486,6 +487,7 @@ private static PluginDrivenExternalTable table(boolean parallelWrite, boolean re ConnectorWritePlanContext writePlanContext = Mockito.mock(ConnectorWritePlanContext.class); Mockito.when(writePlanContext.requiresParallelWrite()).thenReturn(parallelWrite); Mockito.when(writePlanContext.requiresPartitionLocalSort()).thenReturn(requirePartitionSort); + Mockito.when(writePlanContext.getBeExecVersion()).thenReturn(Config.max_be_exec_version); Mockito.when(writePlanContext.getDistribution()).thenReturn(Optional.empty()); Mockito.when(table.resolveWritePlanContext()).thenReturn(writePlanContext); Mockito.when(table.getPartitionColumns()).thenReturn(partitionColumns); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java index 4f80f34b6fe393..120bfbb2c96eb4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java @@ -19,11 +19,13 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.connector.ConnectorSessionBuilder; import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; import org.apache.doris.connector.spi.handle.ConnectorWriteHandle; +import org.apache.doris.connector.spi.handle.WriteOperation; import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot; import org.apache.doris.connector.spi.write.ConnectorSinkPlan; import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider; @@ -109,7 +111,8 @@ public void branchTargetUsesItsExactVersionAwareSnapshotPin() throws AnalysisExc connectorSnapshot, Collections.emptyMap(), Collections.emptyMap()); Mockito.when(metadata.applySnapshot(session, baseHandle, connectorSnapshot)).thenReturn(pinnedHandle); PluginDrivenTableSink sink = new PluginDrivenTableSink(table, provider, session, baseHandle, - Collections.emptyList(), null, null, false, metadata); + Collections.emptyList(), Collections.emptyList(), null, WriteOperation.INSERT, + true, false, null, metadata, Config.max_be_exec_version); PluginDrivenInsertCommandContext ctx = new PluginDrivenInsertCommandContext(); ctx.setBranchName(Optional.of("audit")); @@ -135,7 +138,9 @@ private static PluginDrivenTableSink newPlanProviderSink(ConnectorWritePlanProvi ConnectorTableHandle tableHandle = new ConnectorTableHandle() { }; // targetTable is unused on the plan-provider bind path; pass null to avoid building a // full PluginDrivenExternalTable (which would require a catalog + database). - return new PluginDrivenTableSink(null, provider, session, tableHandle, Collections.emptyList()); + return new PluginDrivenTableSink(null, provider, session, tableHandle, + Collections.emptyList(), Collections.emptyList(), null, WriteOperation.INSERT, + true, false, null, null, Config.max_be_exec_version); } /** Records the bound {@link ConnectorWriteHandle} that the sink hands to {@code planWrite}. */ diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkTest.java index a8b9c5e6b1174c..16225ef85fa01f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkTest.java @@ -20,6 +20,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.connector.spi.ConnectorColumn; +import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.ConnectorType; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; @@ -56,6 +57,86 @@ */ public class PluginDrivenTableSinkTest { + private static PluginDrivenTableSink newSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider provider, ConnectorSession session, + ConnectorTableHandle tableHandle, List columns) { + return newSink(targetTable, provider, session, tableHandle, columns, + columns, null, WriteOperation.INSERT, true, false, null, null); + } + + private static PluginDrivenTableSink newSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider provider, ConnectorSession session, + ConnectorTableHandle tableHandle, List columns, TSortInfo sortInfo) { + return newSink(targetTable, provider, session, tableHandle, columns, + columns, sortInfo, WriteOperation.INSERT, true, false, null, null); + } + + private static PluginDrivenTableSink newSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider provider, ConnectorSession session, + ConnectorTableHandle tableHandle, List columns, TSortInfo sortInfo, + WriteOperation operation) { + return newSink(targetTable, provider, session, tableHandle, columns, + columns, sortInfo, operation, true, false, null, null); + } + + private static PluginDrivenTableSink newSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider provider, ConnectorSession session, + ConnectorTableHandle tableHandle, List columns, TSortInfo sortInfo, + WriteOperation operation, boolean writesDataFiles, boolean requireMergeCardinalityCheck) { + return newSink(targetTable, provider, session, tableHandle, columns, + columns, sortInfo, operation, writesDataFiles, requireMergeCardinalityCheck, null, null); + } + + private static PluginDrivenTableSink newSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider provider, ConnectorSession session, + ConnectorTableHandle tableHandle, List columns, + List boundTargetColumns, TSortInfo sortInfo, + WriteOperation operation, boolean requireMergeCardinalityCheck) { + return newSink(targetTable, provider, session, tableHandle, columns, + boundTargetColumns, sortInfo, operation, true, requireMergeCardinalityCheck, null, null); + } + + private static PluginDrivenTableSink newSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider provider, ConnectorSession session, + ConnectorTableHandle tableHandle, List columns, + List boundTargetColumns, TSortInfo sortInfo, + WriteOperation operation, boolean requireMergeCardinalityCheck, String metadataIdentity) { + return newSink(targetTable, provider, session, tableHandle, columns, + boundTargetColumns, sortInfo, operation, true, requireMergeCardinalityCheck, + metadataIdentity, null); + } + + private static PluginDrivenTableSink newSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider provider, ConnectorSession session, + ConnectorTableHandle tableHandle, List columns, + TSortInfo sortInfo, WriteOperation operation, boolean requireMergeCardinalityCheck, + ConnectorMetadata metadata) { + return newSink(targetTable, provider, session, tableHandle, columns, + columns, sortInfo, operation, true, requireMergeCardinalityCheck, null, metadata); + } + + private static PluginDrivenTableSink newSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider provider, ConnectorSession session, + ConnectorTableHandle tableHandle, List columns, + List boundTargetColumns, TSortInfo sortInfo, + WriteOperation operation, boolean writesDataFiles, boolean requireMergeCardinalityCheck, + String metadataIdentity, ConnectorMetadata metadata) { + return newSink(targetTable, provider, session, tableHandle, columns, boundTargetColumns, + sortInfo, operation, writesDataFiles, requireMergeCardinalityCheck, + metadataIdentity, metadata, Config.max_be_exec_version); + } + + private static PluginDrivenTableSink newSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider provider, ConnectorSession session, + ConnectorTableHandle tableHandle, List columns, + List boundTargetColumns, TSortInfo sortInfo, + WriteOperation operation, boolean writesDataFiles, boolean requireMergeCardinalityCheck, + String metadataIdentity, ConnectorMetadata metadata, int beExecVersion) { + return new PluginDrivenTableSink(targetTable, provider, session, tableHandle, columns, + boundTargetColumns, sortInfo, operation, writesDataFiles, requireMergeCardinalityCheck, + metadataIdentity, metadata, beExecVersion); + } + /** Hand-written {@link ConnectorWritePlanProvider} double recording the delegated call. */ private static final class RecordingWritePlanProvider implements ConnectorWritePlanProvider { private final ConnectorSinkPlan plan; @@ -93,7 +174,7 @@ public void bindDataSinkDelegatesToWritePlanProvider() throws AnalysisException // resolves table facts from its own tableHandle), so a unit of the delegation needs // no full PluginDrivenExternalTable. PluginDrivenTableSink sink = - new PluginDrivenTableSink(null, provider, null, tableHandle, columns); + newSink(null, provider, null, tableHandle, columns); sink.bindDataSink(Optional.empty()); // The connector-built opaque sink is adopted verbatim. @@ -108,6 +189,27 @@ public void bindDataSinkDelegatesToWritePlanProvider() throws AnalysisException Assertions.assertNull(provider.seenHandle.getSortInfo()); } + @Test + public void bindDataSinkKeepsStatementExecutionVersion() throws AnalysisException { + int original = Config.be_exec_version; + try { + int statementVersion = Config.PAIMON_WRITE_MIN_BE_EXEC_VERSION; + RecordingWritePlanProvider provider = new RecordingWritePlanProvider( + new ConnectorSinkPlan(new TDataSink(TDataSinkType.PAIMON_TABLE_SINK))); + PluginDrivenTableSink sink = newSink( + null, provider, null, new ConnectorTableHandle() { }, Collections.emptyList(), + Collections.emptyList(), null, WriteOperation.INSERT, true, false, + null, null, statementVersion); + + Config.be_exec_version = statementVersion - 1; + sink.bindDataSink(Optional.empty()); + + Assertions.assertEquals(statementVersion, provider.seenHandle.getBeExecVersion()); + } finally { + Config.be_exec_version = original; + } + } + @Test public void bindDataSinkKeepsWriteSubsetSeparateFromBoundTargetSchema() throws AnalysisException { RecordingWritePlanProvider provider = new RecordingWritePlanProvider( @@ -117,7 +219,7 @@ public void bindDataSinkKeepsWriteSubsetSeparateFromBoundTargetSchema() throws A List writeColumns = Collections.singletonList(id); List boundTargetColumns = java.util.Arrays.asList(id, name); - PluginDrivenTableSink sink = new PluginDrivenTableSink( + PluginDrivenTableSink sink = newSink( null, provider, null, new ConnectorTableHandle() { }, writeColumns, boundTargetColumns, null, WriteOperation.INSERT, false); sink.bindDataSink(Optional.empty()); @@ -139,7 +241,7 @@ public void bindDataSinkThreadsEngineBuiltWriteSortInfoToHandle() throws Analysi RecordingWritePlanProvider provider = new RecordingWritePlanProvider( new ConnectorSinkPlan(new TDataSink(TDataSinkType.ICEBERG_TABLE_SINK))); - PluginDrivenTableSink sink = new PluginDrivenTableSink( + PluginDrivenTableSink sink = newSink( null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), engineBuilt); sink.bindDataSink(Optional.empty()); @@ -152,7 +254,7 @@ public void bindDataSinkThreadsBoundWriteMetadataIdentityToHandle() throws Analy new ConnectorSinkPlan(new TDataSink(TDataSinkType.ICEBERG_TABLE_SINK))); String metadataIdentity = "sort-generation-3/spec-generation-7"; - PluginDrivenTableSink sink = new PluginDrivenTableSink( + PluginDrivenTableSink sink = newSink( null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), Collections.emptyList(), null, WriteOperation.INSERT, false, metadataIdentity); sink.bindDataSink(Optional.empty()); @@ -172,7 +274,7 @@ public void bindDataSinkThreadsBranchNameToHandle() throws AnalysisException { PluginDrivenInsertCommandContext ctx = new PluginDrivenInsertCommandContext(); ctx.setBranchName(Optional.of("br_1")); - PluginDrivenTableSink sink = new PluginDrivenTableSink( + PluginDrivenTableSink sink = newSink( null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>()); sink.bindDataSink(Optional.of(ctx)); @@ -199,7 +301,7 @@ public void appendExplainInfo(StringBuilder output, String prefix, } }; - PluginDrivenTableSink sink = new PluginDrivenTableSink( + PluginDrivenTableSink sink = newSink( targetTable, provider, null, tableHandle, new ArrayList<>()); String explain = sink.getExplainString("", TExplainLevel.NORMAL); @@ -215,13 +317,13 @@ public void appendExplainInfo(StringBuilder output, String prefix, @Test public void bindDataSinkDefaultsWriteOperationToInsert() throws AnalysisException { - // WHY: a plain INSERT sink (the 5-arg / 6-arg ctors) must keep WriteOperation.INSERT on the write + // WHY: a plain INSERT sink must keep WriteOperation.INSERT on the write // handle so the connector's planWrite stays on the byte-identical INSERT path (TIcebergTableSink, // promoted to OVERWRITE only via isOverwrite()). This pins the dormant-default that guarantees // pre-flip parity for every existing write-capable connector (jdbc / maxcompute / iceberg INSERT). RecordingWritePlanProvider provider = new RecordingWritePlanProvider( new ConnectorSinkPlan(new TDataSink(TDataSinkType.MAXCOMPUTE_TABLE_SINK))); - PluginDrivenTableSink sink = new PluginDrivenTableSink( + PluginDrivenTableSink sink = newSink( null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>()); sink.bindDataSink(Optional.empty()); @@ -237,7 +339,7 @@ public void bindDataSinkThreadsMergeWriteOperationToHandle() throws AnalysisExce // and a MERGE would write as an append. MUTATION: thread INSERT here -> assertion red. RecordingWritePlanProvider provider = new RecordingWritePlanProvider( new ConnectorSinkPlan(new TDataSink(TDataSinkType.ICEBERG_MERGE_SINK))); - PluginDrivenTableSink sink = new PluginDrivenTableSink( + PluginDrivenTableSink sink = newSink( null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), null, WriteOperation.MERGE); sink.bindDataSink(Optional.empty()); @@ -252,7 +354,7 @@ public void bindDataSinkThreadsDeleteWriteOperationToHandle() throws AnalysisExc // hazard as MERGE. MUTATION: thread INSERT here -> assertion red. RecordingWritePlanProvider provider = new RecordingWritePlanProvider( new ConnectorSinkPlan(new TDataSink(TDataSinkType.ICEBERG_DELETE_SINK))); - PluginDrivenTableSink sink = new PluginDrivenTableSink( + PluginDrivenTableSink sink = newSink( null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), null, WriteOperation.DELETE); sink.bindDataSink(Optional.empty()); @@ -264,7 +366,7 @@ null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), public void bindDataSinkThreadsDeleteOnlyMergeToHandle() throws AnalysisException { RecordingWritePlanProvider provider = new RecordingWritePlanProvider( new ConnectorSinkPlan(new TDataSink(TDataSinkType.ICEBERG_MERGE_SINK))); - PluginDrivenTableSink sink = new PluginDrivenTableSink( + PluginDrivenTableSink sink = newSink( null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), null, WriteOperation.MERGE, false, true); sink.bindDataSink(Optional.empty()); @@ -276,33 +378,27 @@ null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), @Test public void deleteOnlyMergeVersionFenceAppliesOnlyToVariantSchemas() throws AnalysisException { - int original = Config.be_exec_version; - try { - Config.be_exec_version = 11; - RecordingWritePlanProvider provider = new RecordingWritePlanProvider( - new ConnectorSinkPlan(new TDataSink(TDataSinkType.ICEBERG_MERGE_SINK))); - ConnectorColumn id = new ConnectorColumn( - "id", ConnectorType.of("INT"), null, true, null); - PluginDrivenTableSink nonVariantSink = new PluginDrivenTableSink( - null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), - Collections.singletonList(id), null, WriteOperation.MERGE, false, true, null, null); - - nonVariantSink.bindDataSink(Optional.empty()); - Assertions.assertNotNull(provider.seenHandle); - Assertions.assertEquals(11, provider.seenHandle.getBeExecVersion()); - - ConnectorColumn payload = new ConnectorColumn( - "payload", ConnectorType.of("VARIANT_COMPUTE_V2"), null, true, null); - PluginDrivenTableSink variantSink = new PluginDrivenTableSink( - null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), - Collections.singletonList(payload), null, WriteOperation.MERGE, false, true, null, null); - - AnalysisException exception = Assertions.assertThrows(AnalysisException.class, - () -> variantSink.bindDataSink(Optional.empty())); - Assertions.assertTrue(exception.getMessage().contains("rolling upgrade")); - } finally { - Config.be_exec_version = original; - } + RecordingWritePlanProvider provider = new RecordingWritePlanProvider( + new ConnectorSinkPlan(new TDataSink(TDataSinkType.ICEBERG_MERGE_SINK))); + ConnectorColumn id = new ConnectorColumn( + "id", ConnectorType.of("INT"), null, true, null); + PluginDrivenTableSink nonVariantSink = newSink( + null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), + Collections.singletonList(id), null, WriteOperation.MERGE, false, true, null, null, 11); + + nonVariantSink.bindDataSink(Optional.empty()); + Assertions.assertNotNull(provider.seenHandle); + Assertions.assertEquals(11, provider.seenHandle.getBeExecVersion()); + + ConnectorColumn payload = new ConnectorColumn( + "payload", ConnectorType.of("VARIANT_COMPUTE_V2"), null, true, null); + PluginDrivenTableSink variantSink = newSink( + null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), + Collections.singletonList(payload), null, WriteOperation.MERGE, false, true, null, null, 11); + + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> variantSink.bindDataSink(Optional.empty())); + Assertions.assertTrue(exception.getMessage().contains("rolling upgrade")); } @Test @@ -315,7 +411,7 @@ public void getExplainStringThreadsWriteOperationToHandle() { Mockito.when(targetTable.getName()).thenReturn("t1"); RecordingWritePlanProvider provider = new RecordingWritePlanProvider( new ConnectorSinkPlan(new TDataSink(TDataSinkType.ICEBERG_MERGE_SINK))); - PluginDrivenTableSink sink = new PluginDrivenTableSink( + PluginDrivenTableSink sink = newSink( targetTable, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), null, WriteOperation.MERGE); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/AbstractJobProcessorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/AbstractJobProcessorTest.java index d20ad967a4ed96..db16588fc4a45e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/AbstractJobProcessorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/AbstractJobProcessorTest.java @@ -23,6 +23,7 @@ import org.apache.doris.qe.runtime.MultiFragmentsPipelineTask; import org.apache.doris.qe.runtime.PipelineExecutionTask; import org.apache.doris.qe.runtime.SingleFragmentPipelineTask; +import org.apache.doris.thrift.TPaimonCommitMessage; import org.apache.doris.thrift.TReportExecStatusParams; import org.apache.doris.thrift.TStatus; import org.apache.doris.thrift.TStatusCode; @@ -31,7 +32,6 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import java.nio.ByteBuffer; import java.util.Collections; import java.util.Map; import java.util.Optional; @@ -66,12 +66,13 @@ void fragmentDispatchBeforeFinishBroadcastsWhenExecutionFinishes() { } @Test - void opaqueConnectorDataRequiresARegisteredFragmentHandler() { + void paimonCommitDataRequiresARegisteredFragmentHandler() { TestJobProcessor processor = new TestJobProcessor(Mockito.mock(CoordinatorContext.class)); processor.setBackendFragmentTasks(Collections.emptyMap()); TReportExecStatusParams params = new TReportExecStatusParams() .setStatus(new TStatus(TStatusCode.OK)) - .setConnectorCommitData(Collections.singletonList(ByteBuffer.wrap(new byte[] {1}))); + .setPaimonCommitMessages(Collections.singletonList( + new TPaimonCommitMessage().setPayload(new byte[] {1}))); Assertions.assertThrows(IllegalStateException.class, () -> processor.updateFragmentExecStatus(params)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorBackendSelectionTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorBackendSelectionTest.java index f0a97a90850886..6db5de57be16d8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorBackendSelectionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorBackendSelectionTest.java @@ -18,6 +18,7 @@ package org.apache.doris.qe; import org.apache.doris.analysis.DescriptorTable; +import org.apache.doris.common.Config; import org.apache.doris.common.Reference; import org.apache.doris.common.UserException; import org.apache.doris.planner.OlapTableSink; @@ -30,6 +31,7 @@ import org.apache.doris.resource.spi.BackendSelectionProvider; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TQueryOptions; import org.apache.doris.thrift.TQueryType; import org.apache.doris.thrift.TScanRangeLocation; import org.apache.doris.thrift.TScanRangeLocations; @@ -60,6 +62,21 @@ void resetBackendSelectionProvider() { BackendSelectionManager.resetProviderForTest(); } + @Test + void testLegacyCoordinatorKeepsPlannerExecutionVersion() { + int original = Config.be_exec_version; + try { + int plannerVersion = Config.max_be_exec_version; + Config.be_exec_version = Config.min_be_exec_version; + + Coordinator coordinator = coordinator(context(1L), Collections.emptyList(), plannerVersion); + + Assertions.assertEquals(plannerVersion, coordinator.getQueryOptions().getBeExecVersion()); + } finally { + Config.be_exec_version = original; + } + } + @Test void testCoordinatorQuerySelectionUsesCoordinatorContext() throws Exception { ConnectContext context = context(3L); @@ -337,10 +354,18 @@ private Map privateMap(Coordinator coordinator, String fieldName) t } private Coordinator coordinator(ConnectContext context, List fragments) { + return coordinator(context, fragments, Config.be_exec_version); + } + + private Coordinator coordinator(ConnectContext context, List fragments, + int beExecVersion) { Planner planner = Mockito.mock(Planner.class); Mockito.when(planner.getFragments()).thenReturn(fragments); Mockito.when(planner.getScanNodes()).thenReturn(Collections.emptyList()); Mockito.when(planner.getDescTable()).thenReturn(new DescriptorTable()); + TQueryOptions queryOptions = context.getSessionVariable().toThrift(); + queryOptions.setBeExecVersion(beExecVersion); + Mockito.when(planner.getQueryOptions()).thenReturn(queryOptions); return new Coordinator(context, planner); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorContextTest.java index 9ecb6e7f68bce5..c25025ebea4222 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorContextTest.java @@ -17,8 +17,10 @@ package org.apache.doris.qe; +import org.apache.doris.common.Config; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.thrift.TQueryGlobals; +import org.apache.doris.thrift.TQueryOptions; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -102,4 +104,20 @@ void testLoadQueryGlobalsSetNanoseconds() { Assertions.assertEquals("UTC", queryGlobals.getTimeZone()); Assertions.assertTrue(queryGlobals.isLoadZeroTolerance()); } + + @Test + void testQueryOptionsKeepPlanningBeExecVersion() { + int planningBeExecVersion = Config.ALIGNED_EXTERNAL_TABLE_SINK_THRIFT_MIN_BE_EXEC_VERSION; + int originalBeExecVersion = Config.be_exec_version; + try { + Config.be_exec_version = planningBeExecVersion - 1; + + TQueryOptions queryOptions = CoordinatorContext.initQueryOptions( + new ConnectContext(), planningBeExecVersion); + + Assertions.assertEquals(planningBeExecVersion, queryOptions.getBeExecVersion()); + } finally { + Config.be_exec_version = originalBeExecVersion; + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java index ad268080e20b57..c4abc116445395 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java @@ -22,6 +22,7 @@ import org.apache.doris.system.Backend; import org.apache.doris.thrift.TIcebergCommitData; import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TPaimonCommitMessage; import org.apache.doris.thrift.TQueryOptions; import org.apache.doris.thrift.TReportExecStatusParams; import org.apache.doris.thrift.TReportExecStatusResult; @@ -37,7 +38,6 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; -import java.nio.ByteBuffer; import java.util.Collections; class QeProcessorImplReportAckTest { @@ -59,10 +59,11 @@ void rejectsExternalReportWithoutCoordinator() { } @Test - void rejectsOpaqueConnectorReportWithoutCoordinator() { + void rejectsPaimonReportWithoutCoordinator() { TReportExecStatusParams params = params(new TUniqueId(12345, 6)); params.unsetIcebergCommitDatas(); - params.setConnectorCommitData(Collections.singletonList(ByteBuffer.wrap(new byte[] {1}))); + params.setPaimonCommitMessages(Collections.singletonList( + new TPaimonCommitMessage().setPayload(new byte[] {1}))); TReportExecStatusResult result = report(params); @@ -113,13 +114,14 @@ void retriesAcceptedExternalReportAfterCoordinatorRemoval() throws Exception { } @Test - void retriesAcceptedOpaqueConnectorReportAfterCoordinatorRemoval() throws Exception { + void retriesAcceptedPaimonReportAfterCoordinatorRemoval() throws Exception { TUniqueId queryId = new TUniqueId(12345, 7); Coordinator coordinator = register(queryId); Mockito.when(coordinator.updateFragmentExecStatus(Mockito.any())).thenReturn(true); TReportExecStatusParams params = params(queryId); params.unsetIcebergCommitDatas(); - params.setConnectorCommitData(Collections.singletonList(ByteBuffer.wrap(new byte[] {1}))); + params.setPaimonCommitMessages(Collections.singletonList( + new TPaimonCommitMessage().setPayload(new byte[] {1}))); TReportExecStatusResult first = report(params); QeProcessorImpl.INSTANCE.unregisterQuery(queryId); diff --git a/fe/fe-core/src/test/java/org/apache/doris/transaction/CommitDataSerializerTest.java b/fe/fe-core/src/test/java/org/apache/doris/transaction/CommitDataSerializerTest.java index d30bdd5b44f599..5b9c575be94de8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/transaction/CommitDataSerializerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/transaction/CommitDataSerializerTest.java @@ -21,6 +21,7 @@ import org.apache.doris.thrift.THivePartitionUpdate; import org.apache.doris.thrift.TIcebergCommitData; import org.apache.doris.thrift.TMCCommitData; +import org.apache.doris.thrift.TPaimonCommitMessage; import org.apache.doris.thrift.TReportExecStatusParams; import org.apache.doris.thrift.TUpdateMode; @@ -31,7 +32,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -79,6 +79,10 @@ private static TIcebergCommitData icebergData(String filePath, long rowCount) { .setPartitionValues(Arrays.asList("2026", "06")); } + private static TPaimonCommitMessage paimonData(byte... payload) { + return new TPaimonCommitMessage().setPayload(payload); + } + private static void assertBinaryRoundTrip(TBase original, TBase target) throws Exception { byte[] bytes = new TSerializer(new TBinaryProtocol.Factory()).serialize(original); @@ -96,6 +100,7 @@ public void binaryProtocolRoundTripIsLossless() throws Exception { assertBinaryRoundTrip(mcData("session-1", 42L, "bWMtcGF5bG9hZA=="), new TMCCommitData()); assertBinaryRoundTrip(hiveData("dt=2026-06-06", 7L, "f1", "f2"), new THivePartitionUpdate()); assertBinaryRoundTrip(icebergData("s3://b/data/0.parquet", 11L), new TIcebergCommitData()); + assertBinaryRoundTrip(paimonData((byte) 1, (byte) 2), new TPaimonCommitMessage()); } /** @@ -140,37 +145,7 @@ public void addCommitData(byte[] commitFragment) { } @Test - public void rawFeedPreservesEachBinarySlice() { - List payloads = new ArrayList<>(); - Transaction collector = new Transaction() { - @Override - public void commit() { - throw new UnsupportedOperationException("commit not expected in this test"); - } - - @Override - public void rollback() { - throw new UnsupportedOperationException("rollback not expected in this test"); - } - - @Override - public void addCommitData(byte[] commitFragment) { - payloads.add(commitFragment); - } - }; - ByteBuffer fragment = ByteBuffer.wrap(new byte[] {0, 1, 2, 3}); - fragment.position(1); - fragment.limit(3); - - CommitDataSerializer.feedRaw(collector, Arrays.asList(fragment, ByteBuffer.wrap(new byte[] {4, 5}))); - - Assertions.assertArrayEquals(new byte[] {1, 2}, payloads.get(0)); - Assertions.assertArrayEquals(new byte[] {4, 5}, payloads.get(1)); - Assertions.assertEquals(1, fragment.position()); - } - - @Test - public void reportFeedRecognizesAndDeliversOpaqueConnectorData() { + public void reportFeedRecognizesAndDeliversPaimonData() throws Exception { List payloads = new ArrayList<>(); Transaction collector = new Transaction() { @Override @@ -188,14 +163,17 @@ public void addCommitData(byte[] commitFragment) { payloads.add(commitFragment); } }; + TPaimonCommitMessage input = paimonData((byte) 6, (byte) 7); TReportExecStatusParams report = new TReportExecStatusParams() - .setConnectorCommitData(Arrays.asList(ByteBuffer.wrap(new byte[] {6, 7}))); + .setPaimonCommitMessages(Arrays.asList(input)); Assertions.assertTrue(CommitDataSerializer.hasCommitData(report)); CommitDataSerializer.feed(collector, report); Assertions.assertEquals(1, payloads.size()); - Assertions.assertArrayEquals(new byte[] {6, 7}, payloads.get(0)); + TPaimonCommitMessage roundTripped = new TPaimonCommitMessage(); + new TDeserializer(new TBinaryProtocol.Factory()).deserialize(roundTripped, payloads.get(0)); + Assertions.assertEquals(input, roundTripped); } } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 25166c50b70a54..83d6b195fac7b0 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -339,8 +339,8 @@ struct TReportExecStatusParams { 33: optional string first_error_msg - // Opaque, connector-owned commit fragments; FE routes them to the transaction. - 34: optional list connector_commit_data + // Paimon commit messages prepared by BE and committed by FE. + 34: optional list paimon_commit_messages } struct TFeResult { diff --git a/gensrc/thrift/HeartbeatService.thrift b/gensrc/thrift/HeartbeatService.thrift index 74c18acef5f644..d295f0ba17d3de 100644 --- a/gensrc/thrift/HeartbeatService.thrift +++ b/gensrc/thrift/HeartbeatService.thrift @@ -65,6 +65,8 @@ struct TBackendInfo { 8: optional bool is_shutdown 9: optional Types.TPort arrow_flight_sql_port 10: optional i64 be_mem // The physical memory available for use by BE. + // Reserved for the Paimon Rust reader and aligned with its branch-4.1 forward-port. + 11: optional bool supports_paimon_rust_reader // For cloud 1000: optional i64 fragment_executing_count 1001: optional i64 fragment_last_active_time diff --git a/gensrc/thrift/Partitions.thrift b/gensrc/thrift/Partitions.thrift index 95ff0eb9487af5..c753f023fc50ce 100644 --- a/gensrc/thrift/Partitions.thrift +++ b/gensrc/thrift/Partitions.thrift @@ -45,20 +45,21 @@ enum TPartitionType { // used for shuffle data by parititon and tablet OLAP_TABLE_SINK_HASH_PARTITIONED = 6, - // used for shuffle data by hive parititon - HIVE_TABLE_SINK_HASH_PARTITIONED = 7, + // used for shuffle data by external table sink ownership key + EXTERNAL_TABLE_SINK_HASH_PARTITIONED = 7, - // used for hive unparititoned table - HIVE_TABLE_SINK_UNPARTITIONED = 8, + // adaptive writer distribution without an ownership key + EXTERNAL_TABLE_SINK_UNPARTITIONED = 8, // used for merge partitioning: insert by partition columns, delete by row_id MERGE_PARTITIONED = 9, - // connector-owned ownership function followed by writer assignment - EXTERNAL_TABLE_SINK_HASH_PARTITIONED = 10, + // used for shuffle data by hive parititon. These master-only values were moved + // before release so values 7 and 8 keep the branch-4.1 external-sink meaning. + HIVE_TABLE_SINK_HASH_PARTITIONED = 10, - // adaptive writer distribution without an ownership key - EXTERNAL_TABLE_SINK_UNPARTITIONED = 11 + // used for hive unparititoned table + HIVE_TABLE_SINK_UNPARTITIONED = 11 } enum TLocalPartitionType { @@ -205,12 +206,35 @@ enum TExternalTableSinkWriterAssignment { SKEWED = 1 } -// FE treats partition_function and its options as opaque connector-owned data. -// BE validates the named function before processing rows. +// Deprecated branch-4.1 routing algorithm. Kept at its original field id in +// TExternalTableSinkHashPartitionInfo for wire compatibility. +enum TExternalTableSinkHashAlgorithm { + DIRECT_HASH = 0, + ICEBERG_TRANSFORM = 1, + PAIMON_FIXED_BUCKET = 2 +} + +// Deprecated branch-4.1 Paimon routing metadata. The generic connector-owned +// partition function encodes this information in partition_function_options. +struct TPaimonFixedBucketInfo { + 1: required i32 num_buckets + 2: required list partition_field_indexes + 3: required list bucket_field_indexes +} + +// Fields 1-4 retain their branch-4.1 meanings and are deprecated on master. +// Master appends its generic connector-owned function representation instead +// of reusing an existing field id for a different type or meaning. struct TExternalTableSinkHashPartitionInfo { - 1: required string partition_function - 2: optional map partition_function_options + 1: optional TExternalTableSinkHashAlgorithm algorithm + 2: optional list partition_transforms 3: required TExternalTableSinkWriterAssignment writer_assignment + 4: optional TPaimonFixedBucketInfo paimon_fixed_bucket_info + + // FE treats these fields as opaque connector-owned data. BE validates the + // named function before processing rows. + 5: optional string partition_function + 6: optional map partition_function_options } // Specification of how a single logical data stream is partitioned. diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 9699414e9f5b8b..35b8b69b69198c 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -373,6 +373,7 @@ enum TPaimonReaderType { PAIMON_JNI = 1, // Deprecated wire value kept during rolling upgrades. New plans never emit it. PAIMON_CPP = 2, + PAIMON_RUST = 3, } struct TPaimonFileDesc { @@ -394,9 +395,13 @@ struct TPaimonFileDesc { 16: optional i64 schema_id; // for schema change. // Reader implementation for logical paimon split. Native file split uses range format type. 17: optional TPaimonReaderType reader_type; + // Reserved for the Paimon Rust reader. Keep these ids aligned with branch-4.1. + 18: optional string paimon_table_schema_json + 19: optional string paimon_branch // Original Paimon RawFile.path() before Doris storage path normalization. Native readers use this - // to materialize the public file-location metadata column. - 18: optional string original_file_path; + // to materialize the public file-location metadata column. This master-only field is appended after + // the branch-4.1 fields instead of reusing field 18. + 20: optional string original_file_path; } struct TTrinoConnectorFileDesc {