Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion be/src/agent/be_exec_version_manager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,11 @@ void BeExecVersionManager::check_function_compatibility(int current_be_exec_vers
// a. use new fixed object serialization way.
// 11: start from master
// a. enforce Iceberg SQL MERGE cardinality only when every executing BE supports it.
// 12: start from Doris 4.1
Comment thread
suxiaogang223 marked this conversation as resolved.
Resolved
// a. support strict ownership hash routing for external table sink writers.
// b. support Paimon default fixed-bucket routing in the external sink exchange.

const int BeExecVersionManager::max_be_exec_version = 11;
const int BeExecVersionManager::max_be_exec_version = 12;
const int BeExecVersionManager::min_be_exec_version = 0;
std::map<std::string, std::set<int>> BeExecVersionManager::_function_change_map {};
std::set<std::string> BeExecVersionManager::_function_restrict_map;
Expand Down
1 change: 1 addition & 0 deletions be/src/agent/be_exec_version_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ constexpr inline int USE_CONST_SERDE =
8; // support const column in serialize/deserialize function: PR #41175
constexpr inline int USE_NEW_FIXED_OBJECT_SERIALIZATION_VERSION = 10;
constexpr inline int SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION = 11;
constexpr inline int SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION = 12;

class BeExecVersionManager {
public:
Expand Down
62 changes: 36 additions & 26 deletions be/src/exec/operator/exchange_sink_operator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,17 @@
#include <random>
#include <string>

#include "agent/be_exec_version_manager.h"
#include "common/status.h"
#include "core/column/column_const.h"
#include "exec/exchange/exchange_writer.h"
#include "exec/exchange/local_exchange_sink_operator.h"
#include "exec/operator/exchange_sink_buffer.h"
#include "exec/operator/operator.h"
#include "exec/operator/sort_source_operator.h"
#include "exec/partitioner/external/external_table_sink_hash_partitioner.h"
#include "exec/pipeline/dependency.h"
#include "exec/pipeline/pipeline_fragment_context.h"
#include "exec/sink/scale_writer_partitioning_exchanger.hpp"
#include "exec/sink/tablet_sink_hash_partitioner.h"
#include "exprs/vexpr.h"
#include "format/transformer/merge_partitioner.h"
Expand Down Expand Up @@ -100,7 +101,7 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf
_part_type = p._part_type;
// Shuffle the channels randomly
if (_part_type == TPartitionType::UNPARTITIONED || _part_type == TPartitionType::RANDOM ||
_part_type == TPartitionType::HIVE_TABLE_SINK_UNPARTITIONED) {
_part_type == TPartitionType::EXTERNAL_TABLE_SINK_UNPARTITIONED) {
std::random_device rd;
std::mt19937 g(rd());
shuffle(channels.begin(), channels.end(), g);
Expand Down Expand Up @@ -155,28 +156,32 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf
p._tablet_sink_partition, p._tablet_sink_location, p._tablet_sink_tuple_id, this);
RETURN_IF_ERROR(_partitioner->init({}));
RETURN_IF_ERROR(_partitioner->prepare(state, {}));
} else if (_part_type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED) {
_partition_count =
channels.size() * config::table_sink_partition_write_max_partition_nums_per_writer;
_partitioner = std::make_unique<ScaleWriterPartitioner>(
channels.size(), _partition_count, channels.size(), 1,
config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold /
state->task_num() ==
0
? config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold
: config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold /
state->task_num(),
config::table_sink_partition_write_min_data_processed_rebalance_threshold /
state->task_num() ==
0
? config::table_sink_partition_write_min_data_processed_rebalance_threshold
: config::table_sink_partition_write_min_data_processed_rebalance_threshold /
state->task_num());

} else if (_part_type == TPartitionType::EXTERNAL_TABLE_SINK_HASH_PARTITIONED) {
if (state->be_exec_version() < SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION) {
Comment thread
suxiaogang223 marked this conversation as resolved.
Resolved
return Status::NotSupported(
"External table sink hash exchange requires BE execution version {}, actual {}",
SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION, state->be_exec_version());
}
if (!p._has_external_table_sink_hash_partition_info) {
return Status::InternalError("External table sink hash partition info is missing");
}
_partition_count = channels.size();
const bool use_crc32c = _state->query_options().__isset.enable_new_shuffle_hash_method &&
_state->query_options().enable_new_shuffle_hash_method;
const ShuffleHashMethod hash_method =
use_crc32c ? ShuffleHashMethod::CRC32C : ShuffleHashMethod::CRC32;
_partitioner = std::make_unique<ExternalTableSinkHashPartitioner>(
_partition_count, hash_method, p._external_table_sink_hash_partition_info);
RETURN_IF_ERROR(_partitioner->init(p._texprs));
RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc));
custom_profile()->add_info_string(
"Partitioner", fmt::format("ScaleWriterPartitioner({})", _partition_count));
"Partitioner",
fmt::format("ExternalTableSinkHashPartitioner({})", _partition_count));
custom_profile()->add_info_string(
"WriterAssignment", p._external_table_sink_hash_partition_info.writer_assignment ==
TExternalTableSinkWriterAssignment::IDENTITY
? "IDENTITY"
: "SKEWED");
} else if (_part_type == TPartitionType::MERGE_PARTITIONED) {
if (!p._has_merge_partition_info) {
return Status::InternalError("Merge partition info is missing");
Expand Down Expand Up @@ -270,7 +275,7 @@ Status ExchangeSinkLocalState::open(RuntimeState* state) {

if (_part_type == TPartitionType::HASH_PARTITIONED ||
_part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED ||
_part_type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED ||
_part_type == TPartitionType::EXTERNAL_TABLE_SINK_HASH_PARTITIONED ||
_part_type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED ||
_part_type == TPartitionType::MERGE_PARTITIONED) {
RETURN_IF_ERROR(_partitioner->open(state));
Expand Down Expand Up @@ -314,8 +319,8 @@ ExchangeSinkOperatorX::ExchangeSinkOperatorX(
sink.output_partition.type == TPartitionType::RANGE_PARTITIONED ||
sink.output_partition.type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED ||
sink.output_partition.type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED ||
sink.output_partition.type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED ||
sink.output_partition.type == TPartitionType::HIVE_TABLE_SINK_UNPARTITIONED ||
sink.output_partition.type == TPartitionType::EXTERNAL_TABLE_SINK_HASH_PARTITIONED ||
sink.output_partition.type == TPartitionType::EXTERNAL_TABLE_SINK_UNPARTITIONED ||
sink.output_partition.type == TPartitionType::MERGE_PARTITIONED);
#endif
_name = "ExchangeSinkOperatorX";
Expand All @@ -327,6 +332,11 @@ ExchangeSinkOperatorX::ExchangeSinkOperatorX(
_merge_partition_info = sink.output_partition.merge_partition_info;
_has_merge_partition_info = true;
}
if (sink.output_partition.__isset.external_table_sink_hash_partition_info) {
_external_table_sink_hash_partition_info =
sink.output_partition.external_table_sink_hash_partition_info;
_has_external_table_sink_hash_partition_info = true;
}

if (_part_type != TPartitionType::UNPARTITIONED) {
// if the destinations only one dest, we need to use broadcast
Expand Down Expand Up @@ -526,11 +536,11 @@ Status ExchangeSinkOperatorX::sink_impl(RuntimeState* state, Block* block, bool
(local_state.current_channel_idx + 1) % local_state.channels.size();
} else if (_part_type == TPartitionType::HASH_PARTITIONED ||
_part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED ||
_part_type == TPartitionType::EXTERNAL_TABLE_SINK_HASH_PARTITIONED ||
_part_type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED ||
_part_type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED ||
_part_type == TPartitionType::MERGE_PARTITIONED) {
RETURN_IF_ERROR(local_state._writer->write(state, block, eos));
} else if (_part_type == TPartitionType::HIVE_TABLE_SINK_UNPARTITIONED) {
} else if (_part_type == TPartitionType::EXTERNAL_TABLE_SINK_UNPARTITIONED) {
// Control the number of channels according to the flow, thereby controlling the number of table sink writers.
RETURN_IF_ERROR(send_to_current_channel());
_data_processed += block->bytes();
Expand Down
2 changes: 2 additions & 0 deletions be/src/exec/operator/exchange_sink_operator.h
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,8 @@ class ExchangeSinkOperatorX MOCK_REMOVE(final) : public DataSinkOperatorX<Exchan
RuntimeState* _state = nullptr;

const std::vector<TExpr> _texprs;
TExternalTableSinkHashPartitionInfo _external_table_sink_hash_partition_info;
bool _has_external_table_sink_hash_partition_info = false;
TMergePartitionInfo _merge_partition_info;
bool _has_merge_partition_info = false;

Expand Down
8 changes: 4 additions & 4 deletions be/src/exec/operator/paimon_table_sink_operator.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@ namespace doris {
/// turn owns one IPaimonWriteBackend + IPaimonWriter. Pipeline parallelism
/// determines the number of concurrent Paimon writer sessions per table.
///
/// Partition and bucket routing is performed internally by the Paimon SDK
/// (Java via JNI, or Rust via FFI). Doris does not compute partition values
/// or bucket ids; it passes complete Blocks through the backend to the SDK,
/// where each row is routed via getPartition(row) + getBucket(row).
/// The upstream sink Exchange may reproduce Paimon's stateless HASH_FIXED
/// selector to establish unique writer ownership. The writer still passes
/// complete Blocks to the SDK, which independently computes partition and
/// bucket values for file writing; no routing column is appended to the row.
///
/// This mirrors Iceberg's approach: IcebergTableSinkOperatorX delegates to
/// AsyncWriterSink<VIcebergTableWriter>, with partition routing inside
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "exec/partitioner/external/external_partition_function_factory.h"

#include "common/status.h"
#include "exec/partitioner/external/paimon_fixed_bucket_partition_function.h"
#include "format/transformer/iceberg_partition_function.h"

namespace doris {
#include "common/compile_check_begin.h"

namespace {
bool has_partition_transform_metadata(const TExternalTableSinkHashPartitionInfo& info) {
return info.__isset.partition_transforms;
}

bool has_paimon_metadata(const TExternalTableSinkHashPartitionInfo& info) {
return info.__isset.paimon_fixed_bucket_info;
}

Status create_direct_hash_function(const TExternalTableSinkHashPartitionInfo& info,
PartitionerBase::HashValType logical_partition_count,
ShuffleHashMethod hash_method,
const std::vector<TExpr>& partition_exprs,
std::unique_ptr<PartitionFunction>* partition_function) {
if (has_partition_transform_metadata(info) || has_paimon_metadata(info)) {
return Status::InvalidArgument("Direct external sink hash contains incompatible metadata");
}
auto function = std::make_unique<HashPartitionFunction>(logical_partition_count, hash_method);
RETURN_IF_ERROR(function->init(partition_exprs));
*partition_function = std::move(function);
return Status::OK();
}

Status create_iceberg_function(const TExternalTableSinkHashPartitionInfo& info,
PartitionerBase::HashValType logical_partition_count,
ShuffleHashMethod hash_method,
const std::vector<TExpr>& partition_exprs,
std::unique_ptr<PartitionFunction>* partition_function) {
if (has_paimon_metadata(info)) {
return Status::InvalidArgument(
"Iceberg external sink routing contains incompatible Paimon metadata");
}
if (!info.__isset.partition_transforms) {
return Status::InvalidArgument("Iceberg external sink partition transforms are missing");
}
if (info.partition_transforms.size() != partition_exprs.size()) {
return Status::InvalidArgument(
"External sink partition transform count {} does not match expression count {}",
info.partition_transforms.size(), partition_exprs.size());
}
std::vector<TIcebergPartitionField> fields;
fields.reserve(partition_exprs.size());
for (size_t index = 0; index < partition_exprs.size(); ++index) {
TIcebergPartitionField field;
field.__set_transform(info.partition_transforms[index]);
field.__set_source_expr(partition_exprs[index]);
fields.emplace_back(std::move(field));
}
auto function = std::make_unique<IcebergInsertPartitionFunction>(
logical_partition_count, hash_method, std::vector<TExpr> {}, std::move(fields));
RETURN_IF_ERROR(function->init({}));
*partition_function = std::move(function);
return Status::OK();
}

Status create_paimon_fixed_bucket_function(const TExternalTableSinkHashPartitionInfo& info,
PartitionerBase::HashValType logical_partition_count,
const std::vector<TExpr>& partition_exprs,
std::unique_ptr<PartitionFunction>* partition_function) {
if (!info.__isset.paimon_fixed_bucket_info) {
return Status::InvalidArgument("Paimon fixed-bucket routing metadata is missing");
}
if (has_partition_transform_metadata(info)) {
return Status::InvalidArgument(
"Paimon fixed-bucket routing contains incompatible metadata");
}
auto function = std::make_unique<PaimonFixedBucketPartitionFunction>(
logical_partition_count, info.paimon_fixed_bucket_info);
RETURN_IF_ERROR(function->init(partition_exprs));
*partition_function = std::move(function);
return Status::OK();
}

} // namespace

Status create_external_partition_function(const TExternalTableSinkHashPartitionInfo& partition_info,
PartitionerBase::HashValType logical_partition_count,
ShuffleHashMethod hash_method,
const std::vector<TExpr>& partition_exprs,
std::unique_ptr<PartitionFunction>* partition_function) {
if (partition_function == nullptr) {
return Status::InvalidArgument("External partition function output is null");
}
switch (partition_info.algorithm) {
case TExternalTableSinkHashAlgorithm::DIRECT_HASH:
return create_direct_hash_function(partition_info, logical_partition_count, hash_method,
partition_exprs, partition_function);
case TExternalTableSinkHashAlgorithm::ICEBERG_TRANSFORM:
return create_iceberg_function(partition_info, logical_partition_count, hash_method,
partition_exprs, partition_function);
case TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET:
return create_paimon_fixed_bucket_function(partition_info, logical_partition_count,
partition_exprs, partition_function);
default:
return Status::InvalidArgument("Unsupported external sink hash algorithm {}",
static_cast<int>(partition_info.algorithm));
}
}

#include "common/compile_check_end.h"
} // namespace doris
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once

#include <gen_cpp/Partitions_types.h>

#include <memory>
#include <vector>

#include "exec/partitioner/partitioner.h"

namespace doris {
#include "common/compile_check_begin.h"

// Validates connector-specific routing metadata and creates the matching logical partition
// function. Writer assignment remains the responsibility of ExternalTableSinkHashPartitioner.
Status create_external_partition_function(const TExternalTableSinkHashPartitionInfo& partition_info,
PartitionerBase::HashValType logical_partition_count,
ShuffleHashMethod hash_method,
const std::vector<TExpr>& partition_exprs,
std::unique_ptr<PartitionFunction>* partition_function);

#include "common/compile_check_end.h"
} // namespace doris
Loading
Loading