Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

#include "common/status.h"
#include "exec/partitioner/external/paimon_fixed_bucket_partition_function.h"
#include "format/transformer/iceberg_partition_function.h"

namespace doris {
namespace {
Expand Down Expand Up @@ -48,6 +49,19 @@ Status parse_indexes(std::string_view value, std::vector<int32_t>* const indexes
return Status::OK();
}

Status create_paimon_fixed_bucket_function(
PartitionerBase::HashValType logical_partition_count, int32_t num_buckets,
std::vector<int32_t> partition_indexes, std::vector<int32_t> bucket_indexes,
const std::vector<TExpr>& partition_exprs,
std::unique_ptr<PartitionFunction>* const partition_function) {
auto function = std::make_unique<PaimonFixedBucketPartitionFunction>(
logical_partition_count, num_buckets, std::move(partition_indexes),
std::move(bucket_indexes));
RETURN_IF_ERROR(function->init(partition_exprs));
*partition_function = std::move(function);
return Status::OK();
}

Status create_paimon_fixed_bucket_function(
const TExternalTableSinkHashPartitionInfo& partition_info,
PartitionerBase::HashValType logical_partition_count,
Expand All @@ -69,14 +83,77 @@ Status create_paimon_fixed_bucket_function(
RETURN_IF_ERROR(parse_int32(options.at("num_buckets"), &num_buckets));
RETURN_IF_ERROR(parse_indexes(options.at("partition_field_indexes"), &partition_indexes));
RETURN_IF_ERROR(parse_indexes(options.at("bucket_field_indexes"), &bucket_indexes));
auto function = std::make_unique<PaimonFixedBucketPartitionFunction>(
RETURN_IF_ERROR(create_paimon_fixed_bucket_function(
logical_partition_count, num_buckets, std::move(partition_indexes),
std::move(bucket_indexes));
RETURN_IF_ERROR(function->init(partition_exprs));
*partition_function = std::move(function);
std::move(bucket_indexes), partition_exprs, partition_function));
return Status::OK();
}

Status create_legacy_external_partition_function(
const TExternalTableSinkHashPartitionInfo& partition_info,
PartitionerBase::HashValType logical_partition_count, ShuffleHashMethod hash_method,
const std::vector<TExpr>& partition_exprs,
std::unique_ptr<PartitionFunction>* const partition_function) {
if (!partition_info.__isset.algorithm) {
return Status::InvalidArgument("External sink partition function is missing");
}
switch (partition_info.algorithm) {
case TExternalTableSinkHashAlgorithm::DIRECT_HASH: {
if (partition_info.__isset.partition_transforms ||
partition_info.__isset.paimon_fixed_bucket_info) {
return Status::InvalidArgument(
"Legacy direct external sink hash contains incompatible metadata");
}
auto function =
std::make_unique<HashPartitionFunction>(logical_partition_count, hash_method);
RETURN_IF_ERROR(function->init(partition_exprs));
*partition_function = std::move(function);
return Status::OK();
}
case TExternalTableSinkHashAlgorithm::ICEBERG_TRANSFORM: {
if (partition_info.__isset.paimon_fixed_bucket_info) {
return Status::InvalidArgument(
"Legacy Iceberg sink routing contains incompatible Paimon metadata");
}
if (!partition_info.__isset.partition_transforms ||
partition_info.partition_transforms.size() != partition_exprs.size()) {
return Status::InvalidArgument(
"Legacy Iceberg sink partition transforms must match expression count");
}
std::vector<TIcebergPartitionField> fields;
fields.reserve(partition_exprs.size());
for (size_t index = 0; index < partition_exprs.size(); ++index) {
TIcebergPartitionField field;
field.__set_transform(partition_info.partition_transforms[index]);
field.__set_source_expr(partition_exprs[index]);
fields.emplace_back(std::move(field));
}
auto function = std::make_unique<IcebergInsertPartitionFunction>(
logical_partition_count, hash_method, std::vector<TExpr> {}, std::move(fields));
RETURN_IF_ERROR(function->init({}));
*partition_function = std::move(function);
return Status::OK();
}
case TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET: {
if (partition_info.__isset.partition_transforms) {
return Status::InvalidArgument(
"Legacy Paimon fixed-bucket routing contains incompatible Iceberg metadata");
}
if (!partition_info.__isset.paimon_fixed_bucket_info) {
return Status::InvalidArgument(
"Legacy Paimon fixed-bucket routing metadata is missing");
}
const auto& info = partition_info.paimon_fixed_bucket_info;
return create_paimon_fixed_bucket_function(
logical_partition_count, info.num_buckets, info.partition_field_indexes,
info.bucket_field_indexes, partition_exprs, partition_function);
}
default:
return Status::InvalidArgument("Unsupported legacy external sink hash algorithm {}",
static_cast<int>(partition_info.algorithm));
}
}

} // namespace

Status create_external_partition_function(
Expand All @@ -87,6 +164,11 @@ Status create_external_partition_function(
if (partition_function == nullptr) {
return Status::InvalidArgument("External partition function output is null");
}
if (!partition_info.__isset.partition_function) {
return create_legacy_external_partition_function(partition_info, logical_partition_count,
hash_method, partition_exprs,
partition_function);
}
if (partition_info.partition_function == "paimon_fixed_bucket") {
return create_paimon_fixed_bucket_function(partition_info, logical_partition_count,
partition_exprs, partition_function);
Expand Down
2 changes: 1 addition & 1 deletion be/src/exec/pipeline/pipeline_fragment_context.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2602,7 +2602,7 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r
}

const bool requires_external_file_ack =
params.__isset.iceberg_commit_datas || params.__isset.connector_commit_data;
params.__isset.iceberg_commit_datas || params.__isset.paimon_commit_messages;
Comment thread
suxiaogang223 marked this conversation as resolved.
Resolved
if (rpc_status.ok() && requires_external_file_ack &&
(!res.__isset.external_file_commit_data_accepted ||
!res.external_file_commit_data_accepted)) {
Expand Down
4 changes: 2 additions & 2 deletions be/src/exec/sink/viceberg_delete_sink.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
#include "core/data_type/data_type_string.h"
#include "core/data_type/data_type_struct.h"
#include "exec/common/endian.h"
#include "exec/sink/writer/iceberg/iceberg_writer_compatibility.h"
#include "exec/sink/writer/external_file_writer_compatibility.h"
#include "exprs/vexpr.h"
#include "format/table/deletion_vector.h"
#include "format/table/iceberg_delete_file_reader_helper.h"
Expand Down Expand Up @@ -204,7 +204,7 @@ Status VIcebergDeleteSink::init_properties(ObjectPool* pool) {
Status VIcebergDeleteSink::open(RuntimeState* state, RuntimeProfile* profile) {
_state = state;

RETURN_IF_ERROR(validate_iceberg_external_file_report_ack(state->query_options()));
RETURN_IF_ERROR(validate_external_file_report_ack(state->query_options()));

// Initialize counters
_written_rows_counter = ADD_COUNTER(profile, "RowsWritten", TUnit::UNIT);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,13 @@

namespace doris {

inline Status validate_iceberg_external_file_report_ack(const TQueryOptions& query_options) {
inline Status validate_external_file_report_ack(const TQueryOptions& query_options) {
if (!query_options.__isset.supports_external_file_report_ack ||
!query_options.supports_external_file_report_ack) {
// A pre-ACK coordinator cannot safely take ownership of files created by this sink.
return Status::NotSupported(
"Iceberg writes require a coordinator that acknowledges external-file reports");
"External-file writes require a coordinator that acknowledges external-file "
"reports");
}
return Status::OK();
}
Expand Down
4 changes: 2 additions & 2 deletions be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_struct.h"
#include "core/data_type_serde/data_type_serde.h"
#include "exec/sink/writer/external_file_writer_compatibility.h"
#include "exec/sink/writer/iceberg/iceberg_partition_path.h"
#include "exec/sink/writer/iceberg/iceberg_writer_compatibility.h"
#include "exec/sink/writer/iceberg/partition_transformers.h"
#include "exec/sink/writer/iceberg/viceberg_partition_writer.h"
#include "exec/sink/writer/iceberg/viceberg_sort_writer.h"
Expand All @@ -59,7 +59,7 @@ Status VIcebergTableWriter::open(RuntimeState* state, RuntimeProfile* profile) {
_state = state;
_operator_profile = profile;

RETURN_IF_ERROR(validate_iceberg_external_file_report_ack(state->query_options()));
RETURN_IF_ERROR(validate_external_file_report_ack(state->query_options()));

// Get target file size from query options
// If value is 0 or not set, use config::iceberg_sink_max_file_size
Expand Down
27 changes: 18 additions & 9 deletions be/src/exec/sink/writer/paimon/paimon_table_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@
#include "common/logging.h"
#include "core/block/block.h"
#include "core/block/materialize_block.h"
#include "exec/sink/writer/external_file_writer_compatibility.h"
#include "exprs/vexpr_context.h"
#include "runtime/runtime_state.h"
#include "util/thrift_util.h"

namespace doris {

Expand All @@ -35,6 +35,8 @@ PaimonTableWriter::PaimonTableWriter(TDataSink t_sink, const VExprContextSPtrs&
Status PaimonTableWriter::open(RuntimeState* state, RuntimeProfile* profile) {
_state = state;

RETURN_IF_ERROR(validate_external_file_report_ack(state->query_options()));

// Register profile counters
_written_rows_counter = ADD_COUNTER(profile, "WrittenRows", TUnit::UNIT);
_written_bytes_counter = ADD_COUNTER(profile, "WrittenBytes", TUnit::BYTES);
Expand Down Expand Up @@ -112,6 +114,16 @@ Status PaimonTableWriter::close(Status status) {
}
}

// Reserve the shared report budget while the writer can still abort its prepared files.
ExternalFileReportReservation commit_message_reservation;
if (status.ok()) {
Status reserve_st =
_state->reserve_paimon_commit_messages(messages, &commit_message_reservation);
if (!reserve_st.ok()) {
status = reserve_st;
}
}

// If prepare_commit failed or the incoming status was already an error,
// abort the writer to clean up uncommitted data files.
if (!status.ok()) {
Expand Down Expand Up @@ -148,14 +160,11 @@ Status PaimonTableWriter::close(Status status) {
DORIS_CHECK(msg.__isset.payload);
COUNTER_UPDATE(_commit_payload_bytes_counter, static_cast<int64_t>(msg.payload.size()));
}
ThriftSerializer serializer(false, 256);
for (auto& message : messages) {
std::string commit_data;
RETURN_IF_ERROR(serializer.serialize(&message, &commit_data));
RETURN_IF_ERROR(_state->add_connector_commit_data(std::move(commit_data)));
}
if (!messages.empty()) {
LOG(INFO) << "Paimon writer closed: " << messages.size()
const size_t message_count = messages.size();
_state->publish_reserved_paimon_commit_messages(std::move(messages),
&commit_message_reservation);
if (message_count > 0) {
LOG(INFO) << "Paimon writer closed: " << message_count
<< " commit messages, total rows=" << _written_rows;
}
}
Expand Down
3 changes: 2 additions & 1 deletion be/src/exec/sink/writer/paimon/paimon_table_writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ class RuntimeState;
/// Commit flow (BE only prepares messages; FE is the commit coordinator):
/// close() → writer->prepare_commit()
/// → collect TPaimonCommitMessage[] (DPCM-framed serialized messages)
/// → serialize each message into RuntimeState::add_connector_commit_data()
/// → reserve report budget while abort is still possible
/// → RuntimeState::publish_reserved_paimon_commit_messages()
/// → RPC to FE Coordinator → PaimonTransaction
class PaimonTableWriter final {
public:
Expand Down
60 changes: 49 additions & 11 deletions be/src/runtime/runtime_state.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,13 @@
#include "util/uid_util.h"

namespace doris {

ExternalFileReportReservation::~ExternalFileReportReservation() {
if (_state != nullptr) {
_state->release_external_file_report_reservation(_reserved_bytes);
}
}

using namespace ErrorCode;

Status RuntimeState::add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_data) {
Expand Down Expand Up @@ -84,25 +91,56 @@ Status RuntimeState::add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_
return Status::OK();
}

Status RuntimeState::add_connector_commit_data(std::string commit_data) {
Status RuntimeState::reserve_paimon_commit_messages(
const std::vector<TPaimonCommitMessage>& commit_messages,
ExternalFileReportReservation* reservation) {
DORIS_CHECK(reservation != nullptr);
DORIS_CHECK(reservation->_state == nullptr);
size_t serialized_bytes = 0;
ThriftSerializer serializer(false, 256);
for (const auto& commit_message : commit_messages) {
uint32_t serialized_size = 0;
uint8_t* buffer = nullptr;
RETURN_IF_ERROR(serializer.serialize(&commit_message, &serialized_size, &buffer));
serialized_bytes += serialized_size + sizeof(uint32_t);
}

constexpr size_t report_envelope_headroom = 1024 * 1024;
const size_t thrift_limit = coordinator_thrift_message_limit();
const size_t commit_data_limit =
thrift_limit > report_envelope_headroom ? thrift_limit - report_envelope_headroom : 0;
std::lock_guard<std::mutex> budget_lock(_external_file_report_state->mutex);
if (_external_file_report_state->serialized_commit_bytes + commit_data.size() +
sizeof(uint32_t) >
if (_external_file_report_state->serialized_commit_bytes + serialized_bytes >
Comment thread
suxiaogang223 marked this conversation as resolved.
Resolved
commit_data_limit) {
return Status::InternalError(
"Connector commit metadata exceeds the Thrift report limit; reduce commit "
"metadata size");
"Paimon commit metadata exceeds the Thrift report limit; reduce output file "
"count");
}
std::lock_guard<std::mutex> data_lock(_connector_commit_data_mutex);
_external_file_report_state->serialized_commit_bytes += commit_data.size() + sizeof(uint32_t);
_connector_commit_data.emplace_back(std::move(commit_data));
_external_file_report_state->serialized_commit_bytes += serialized_bytes;
reservation->_state = this;
reservation->_reserved_bytes = serialized_bytes;
return Status::OK();
}

void RuntimeState::publish_reserved_paimon_commit_messages(
std::vector<TPaimonCommitMessage> commit_messages,
ExternalFileReportReservation* reservation) {
DORIS_CHECK(reservation != nullptr);
DORIS_CHECK(reservation->_state == this);
std::lock_guard<std::mutex> data_lock(_paimon_commit_messages_mutex);
for (auto& commit_message : commit_messages) {
_paimon_commit_messages.emplace_back(std::move(commit_message));
}
reservation->_state = nullptr;
reservation->_reserved_bytes = 0;
}

void RuntimeState::release_external_file_report_reservation(size_t reserved_bytes) {
std::lock_guard<std::mutex> budget_lock(_external_file_report_state->mutex);
DORIS_CHECK_GE(_external_file_report_state->serialized_commit_bytes, reserved_bytes);
_external_file_report_state->serialized_commit_bytes -= reserved_bytes;
}

size_t RuntimeState::coordinator_thrift_message_limit() const {
int32_t effective_thrift_limit = std::max(config::thrift_max_message_size, 0);
if (_query_options.__isset.coordinator_thrift_max_message_size &&
Expand Down Expand Up @@ -134,9 +172,9 @@ void RuntimeState::append_external_file_commit_data(TReportExecStatusParams* par
params->mc_commit_datas.insert(params->mc_commit_datas.end(), commit_datas.begin(),
commit_datas.end());
}
append_connector_commit_data(&params->connector_commit_data);
if (!params->connector_commit_data.empty()) {
params->__isset.connector_commit_data = true;
append_paimon_commit_messages(&params->paimon_commit_messages);
if (!params->paimon_commit_messages.empty()) {
params->__isset.paimon_commit_messages = true;
}
}

Expand Down
Loading
Loading