diff --git a/be/src/agent/be_exec_version_manager.cpp b/be/src/agent/be_exec_version_manager.cpp index 4c3bdbd9f31445..de9dfb9745dc5e 100644 --- a/be/src/agent/be_exec_version_manager.cpp +++ b/be/src/agent/be_exec_version_manager.cpp @@ -126,8 +126,11 @@ void BeExecVersionManager::check_function_compatibility(int current_be_exec_vers // a. use new fixed object serialization way. // 11: start from master // a. enforce Iceberg SQL MERGE cardinality only when every executing BE supports it. +// 12: start from Doris 4.1 +// a. support strict ownership hash routing for external table sink writers. +// b. support Paimon default fixed-bucket routing in the external sink exchange. -const int BeExecVersionManager::max_be_exec_version = 11; +const int BeExecVersionManager::max_be_exec_version = 12; const int BeExecVersionManager::min_be_exec_version = 0; std::map> BeExecVersionManager::_function_change_map {}; std::set BeExecVersionManager::_function_restrict_map; diff --git a/be/src/agent/be_exec_version_manager.h b/be/src/agent/be_exec_version_manager.h index f5c16e9a2a55ef..b19d1aabac8c43 100644 --- a/be/src/agent/be_exec_version_manager.h +++ b/be/src/agent/be_exec_version_manager.h @@ -34,6 +34,7 @@ constexpr inline int USE_CONST_SERDE = 8; // support const column in serialize/deserialize function: PR #41175 constexpr inline int USE_NEW_FIXED_OBJECT_SERIALIZATION_VERSION = 10; constexpr inline int SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION = 11; +constexpr inline int SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION = 12; class BeExecVersionManager { public: diff --git a/be/src/exec/operator/exchange_sink_operator.cpp b/be/src/exec/operator/exchange_sink_operator.cpp index 4d767611e1c705..025d6901c0de25 100644 --- a/be/src/exec/operator/exchange_sink_operator.cpp +++ b/be/src/exec/operator/exchange_sink_operator.cpp @@ -28,6 +28,7 @@ #include #include +#include "agent/be_exec_version_manager.h" #include "common/status.h" #include "core/column/column_const.h" #include "exec/exchange/exchange_writer.h" @@ -35,9 +36,9 @@ #include "exec/operator/exchange_sink_buffer.h" #include "exec/operator/operator.h" #include "exec/operator/sort_source_operator.h" +#include "exec/partitioner/external/external_table_sink_hash_partitioner.h" #include "exec/pipeline/dependency.h" #include "exec/pipeline/pipeline_fragment_context.h" -#include "exec/sink/scale_writer_partitioning_exchanger.hpp" #include "exec/sink/tablet_sink_hash_partitioner.h" #include "exprs/vexpr.h" #include "format/transformer/merge_partitioner.h" @@ -100,7 +101,7 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf _part_type = p._part_type; // Shuffle the channels randomly if (_part_type == TPartitionType::UNPARTITIONED || _part_type == TPartitionType::RANDOM || - _part_type == TPartitionType::HIVE_TABLE_SINK_UNPARTITIONED) { + _part_type == TPartitionType::EXTERNAL_TABLE_SINK_UNPARTITIONED) { std::random_device rd; std::mt19937 g(rd()); shuffle(channels.begin(), channels.end(), g); @@ -155,28 +156,32 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf p._tablet_sink_partition, p._tablet_sink_location, p._tablet_sink_tuple_id, this); RETURN_IF_ERROR(_partitioner->init({})); RETURN_IF_ERROR(_partitioner->prepare(state, {})); - } else if (_part_type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED) { - _partition_count = - channels.size() * config::table_sink_partition_write_max_partition_nums_per_writer; - _partitioner = std::make_unique( - channels.size(), _partition_count, channels.size(), 1, - config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold / - state->task_num() == - 0 - ? config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold - : config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold / - state->task_num(), - config::table_sink_partition_write_min_data_processed_rebalance_threshold / - state->task_num() == - 0 - ? config::table_sink_partition_write_min_data_processed_rebalance_threshold - : config::table_sink_partition_write_min_data_processed_rebalance_threshold / - state->task_num()); - + } else if (_part_type == TPartitionType::EXTERNAL_TABLE_SINK_HASH_PARTITIONED) { + if (state->be_exec_version() < SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION) { + return Status::NotSupported( + "External table sink hash exchange requires BE execution version {}, actual {}", + SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION, state->be_exec_version()); + } + if (!p._has_external_table_sink_hash_partition_info) { + return Status::InternalError("External table sink hash partition info is missing"); + } + _partition_count = channels.size(); + const bool use_crc32c = _state->query_options().__isset.enable_new_shuffle_hash_method && + _state->query_options().enable_new_shuffle_hash_method; + const ShuffleHashMethod hash_method = + use_crc32c ? ShuffleHashMethod::CRC32C : ShuffleHashMethod::CRC32; + _partitioner = std::make_unique( + _partition_count, hash_method, p._external_table_sink_hash_partition_info); RETURN_IF_ERROR(_partitioner->init(p._texprs)); RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc)); custom_profile()->add_info_string( - "Partitioner", fmt::format("ScaleWriterPartitioner({})", _partition_count)); + "Partitioner", + fmt::format("ExternalTableSinkHashPartitioner({})", _partition_count)); + custom_profile()->add_info_string( + "WriterAssignment", p._external_table_sink_hash_partition_info.writer_assignment == + TExternalTableSinkWriterAssignment::IDENTITY + ? "IDENTITY" + : "SKEWED"); } else if (_part_type == TPartitionType::MERGE_PARTITIONED) { if (!p._has_merge_partition_info) { return Status::InternalError("Merge partition info is missing"); @@ -270,7 +275,7 @@ Status ExchangeSinkLocalState::open(RuntimeState* state) { if (_part_type == TPartitionType::HASH_PARTITIONED || _part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED || - _part_type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED || + _part_type == TPartitionType::EXTERNAL_TABLE_SINK_HASH_PARTITIONED || _part_type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED || _part_type == TPartitionType::MERGE_PARTITIONED) { RETURN_IF_ERROR(_partitioner->open(state)); @@ -314,8 +319,8 @@ ExchangeSinkOperatorX::ExchangeSinkOperatorX( sink.output_partition.type == TPartitionType::RANGE_PARTITIONED || sink.output_partition.type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED || sink.output_partition.type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED || - sink.output_partition.type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED || - sink.output_partition.type == TPartitionType::HIVE_TABLE_SINK_UNPARTITIONED || + sink.output_partition.type == TPartitionType::EXTERNAL_TABLE_SINK_HASH_PARTITIONED || + sink.output_partition.type == TPartitionType::EXTERNAL_TABLE_SINK_UNPARTITIONED || sink.output_partition.type == TPartitionType::MERGE_PARTITIONED); #endif _name = "ExchangeSinkOperatorX"; @@ -327,6 +332,11 @@ ExchangeSinkOperatorX::ExchangeSinkOperatorX( _merge_partition_info = sink.output_partition.merge_partition_info; _has_merge_partition_info = true; } + if (sink.output_partition.__isset.external_table_sink_hash_partition_info) { + _external_table_sink_hash_partition_info = + sink.output_partition.external_table_sink_hash_partition_info; + _has_external_table_sink_hash_partition_info = true; + } if (_part_type != TPartitionType::UNPARTITIONED) { // if the destinations only one dest, we need to use broadcast @@ -526,11 +536,11 @@ Status ExchangeSinkOperatorX::sink_impl(RuntimeState* state, Block* block, bool (local_state.current_channel_idx + 1) % local_state.channels.size(); } else if (_part_type == TPartitionType::HASH_PARTITIONED || _part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED || + _part_type == TPartitionType::EXTERNAL_TABLE_SINK_HASH_PARTITIONED || _part_type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED || - _part_type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED || _part_type == TPartitionType::MERGE_PARTITIONED) { RETURN_IF_ERROR(local_state._writer->write(state, block, eos)); - } else if (_part_type == TPartitionType::HIVE_TABLE_SINK_UNPARTITIONED) { + } else if (_part_type == TPartitionType::EXTERNAL_TABLE_SINK_UNPARTITIONED) { // Control the number of channels according to the flow, thereby controlling the number of table sink writers. RETURN_IF_ERROR(send_to_current_channel()); _data_processed += block->bytes(); diff --git a/be/src/exec/operator/exchange_sink_operator.h b/be/src/exec/operator/exchange_sink_operator.h index 09f4cf8b047107..59d3f377b6847f 100644 --- a/be/src/exec/operator/exchange_sink_operator.h +++ b/be/src/exec/operator/exchange_sink_operator.h @@ -244,6 +244,8 @@ class ExchangeSinkOperatorX MOCK_REMOVE(final) : public DataSinkOperatorX _texprs; + TExternalTableSinkHashPartitionInfo _external_table_sink_hash_partition_info; + bool _has_external_table_sink_hash_partition_info = false; TMergePartitionInfo _merge_partition_info; bool _has_merge_partition_info = false; diff --git a/be/src/exec/operator/paimon_table_sink_operator.h b/be/src/exec/operator/paimon_table_sink_operator.h index 0bed36a27da75e..044068c01de97f 100644 --- a/be/src/exec/operator/paimon_table_sink_operator.h +++ b/be/src/exec/operator/paimon_table_sink_operator.h @@ -36,10 +36,10 @@ namespace doris { /// turn owns one IPaimonWriteBackend + IPaimonWriter. Pipeline parallelism /// determines the number of concurrent Paimon writer sessions per table. /// -/// Partition and bucket routing is performed internally by the Paimon SDK -/// (Java via JNI, or Rust via FFI). Doris does not compute partition values -/// or bucket ids; it passes complete Blocks through the backend to the SDK, -/// where each row is routed via getPartition(row) + getBucket(row). +/// The upstream sink Exchange may reproduce Paimon's stateless HASH_FIXED +/// selector to establish unique writer ownership. The writer still passes +/// complete Blocks to the SDK, which independently computes partition and +/// bucket values for file writing; no routing column is appended to the row. /// /// This mirrors Iceberg's approach: IcebergTableSinkOperatorX delegates to /// AsyncWriterSink, with partition routing inside diff --git a/be/src/exec/partitioner/external/external_partition_function_factory.cpp b/be/src/exec/partitioner/external/external_partition_function_factory.cpp new file mode 100644 index 00000000000000..790e2efcc8f2bf --- /dev/null +++ b/be/src/exec/partitioner/external/external_partition_function_factory.cpp @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/partitioner/external/external_partition_function_factory.h" + +#include "common/status.h" +#include "exec/partitioner/external/paimon_fixed_bucket_partition_function.h" +#include "format/transformer/iceberg_partition_function.h" + +namespace doris { +#include "common/compile_check_begin.h" + +namespace { +bool has_partition_transform_metadata(const TExternalTableSinkHashPartitionInfo& info) { + return info.__isset.partition_transforms; +} + +bool has_paimon_metadata(const TExternalTableSinkHashPartitionInfo& info) { + return info.__isset.paimon_fixed_bucket_info; +} + +Status create_direct_hash_function(const TExternalTableSinkHashPartitionInfo& info, + PartitionerBase::HashValType logical_partition_count, + ShuffleHashMethod hash_method, + const std::vector& partition_exprs, + std::unique_ptr* partition_function) { + if (has_partition_transform_metadata(info) || has_paimon_metadata(info)) { + return Status::InvalidArgument("Direct external sink hash contains incompatible metadata"); + } + auto function = std::make_unique(logical_partition_count, hash_method); + RETURN_IF_ERROR(function->init(partition_exprs)); + *partition_function = std::move(function); + return Status::OK(); +} + +Status create_iceberg_function(const TExternalTableSinkHashPartitionInfo& info, + PartitionerBase::HashValType logical_partition_count, + ShuffleHashMethod hash_method, + const std::vector& partition_exprs, + std::unique_ptr* partition_function) { + if (has_paimon_metadata(info)) { + return Status::InvalidArgument( + "Iceberg external sink routing contains incompatible Paimon metadata"); + } + if (!info.__isset.partition_transforms) { + return Status::InvalidArgument("Iceberg external sink partition transforms are missing"); + } + if (info.partition_transforms.size() != partition_exprs.size()) { + return Status::InvalidArgument( + "External sink partition transform count {} does not match expression count {}", + info.partition_transforms.size(), partition_exprs.size()); + } + std::vector fields; + fields.reserve(partition_exprs.size()); + for (size_t index = 0; index < partition_exprs.size(); ++index) { + TIcebergPartitionField field; + field.__set_transform(info.partition_transforms[index]); + field.__set_source_expr(partition_exprs[index]); + fields.emplace_back(std::move(field)); + } + auto function = std::make_unique( + logical_partition_count, hash_method, std::vector {}, std::move(fields)); + RETURN_IF_ERROR(function->init({})); + *partition_function = std::move(function); + return Status::OK(); +} + +Status create_paimon_fixed_bucket_function(const TExternalTableSinkHashPartitionInfo& info, + PartitionerBase::HashValType logical_partition_count, + const std::vector& partition_exprs, + std::unique_ptr* partition_function) { + if (!info.__isset.paimon_fixed_bucket_info) { + return Status::InvalidArgument("Paimon fixed-bucket routing metadata is missing"); + } + if (has_partition_transform_metadata(info)) { + return Status::InvalidArgument( + "Paimon fixed-bucket routing contains incompatible metadata"); + } + auto function = std::make_unique( + logical_partition_count, info.paimon_fixed_bucket_info); + RETURN_IF_ERROR(function->init(partition_exprs)); + *partition_function = std::move(function); + return Status::OK(); +} + +} // namespace + +Status create_external_partition_function(const TExternalTableSinkHashPartitionInfo& partition_info, + PartitionerBase::HashValType logical_partition_count, + ShuffleHashMethod hash_method, + const std::vector& partition_exprs, + std::unique_ptr* partition_function) { + if (partition_function == nullptr) { + return Status::InvalidArgument("External partition function output is null"); + } + switch (partition_info.algorithm) { + case TExternalTableSinkHashAlgorithm::DIRECT_HASH: + return create_direct_hash_function(partition_info, logical_partition_count, hash_method, + partition_exprs, partition_function); + case TExternalTableSinkHashAlgorithm::ICEBERG_TRANSFORM: + return create_iceberg_function(partition_info, logical_partition_count, hash_method, + partition_exprs, partition_function); + case TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET: + return create_paimon_fixed_bucket_function(partition_info, logical_partition_count, + partition_exprs, partition_function); + default: + return Status::InvalidArgument("Unsupported external sink hash algorithm {}", + static_cast(partition_info.algorithm)); + } +} + +#include "common/compile_check_end.h" +} // namespace doris diff --git a/be/src/exec/partitioner/external/external_partition_function_factory.h b/be/src/exec/partitioner/external/external_partition_function_factory.h new file mode 100644 index 00000000000000..e397975bbc9b7f --- /dev/null +++ b/be/src/exec/partitioner/external/external_partition_function_factory.h @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include + +#include "exec/partitioner/partitioner.h" + +namespace doris { +#include "common/compile_check_begin.h" + +// Validates connector-specific routing metadata and creates the matching logical partition +// function. Writer assignment remains the responsibility of ExternalTableSinkHashPartitioner. +Status create_external_partition_function(const TExternalTableSinkHashPartitionInfo& partition_info, + PartitionerBase::HashValType logical_partition_count, + ShuffleHashMethod hash_method, + const std::vector& partition_exprs, + std::unique_ptr* partition_function); + +#include "common/compile_check_end.h" +} // namespace doris diff --git a/be/src/exec/partitioner/external/external_table_sink_hash_partitioner.cpp b/be/src/exec/partitioner/external/external_table_sink_hash_partitioner.cpp new file mode 100644 index 00000000000000..b19209d0e29a9c --- /dev/null +++ b/be/src/exec/partitioner/external/external_table_sink_hash_partitioner.cpp @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/partitioner/external/external_table_sink_hash_partitioner.h" + +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "common/status.h" +#include "exec/partitioner/external/external_partition_function_factory.h" + +namespace doris { +#include "common/compile_check_begin.h" + +namespace { +Status logical_partition_count(uint32_t writer_count, + TExternalTableSinkWriterAssignment::type assignment, + uint32_t* result) { + if (writer_count == 0) { + return Status::InvalidArgument("External sink writer count must be positive"); + } + if (assignment != TExternalTableSinkWriterAssignment::SKEWED) { + *result = writer_count; + return Status::OK(); + } + const uint32_t partitions_per_writer = static_cast( + std::max(1, config::table_sink_partition_write_max_partition_nums_per_writer)); + if (writer_count > std::numeric_limits::max() / partitions_per_writer) { + return Status::InvalidArgument("External sink logical partition count overflows"); + } + *result = writer_count * partitions_per_writer; + return Status::OK(); +} +} // namespace + +ExternalTableSinkHashPartitioner::ExternalTableSinkHashPartitioner( + HashValType partition_count, ShuffleHashMethod hash_method, + TExternalTableSinkHashPartitionInfo partition_info) + : PartitionerBase(partition_count), + _hash_method(hash_method), + _partition_info(std::move(partition_info)), + _logical_partition_count(partition_count) {} + +Status ExternalTableSinkHashPartitioner::init(const std::vector& texprs) { + if (!_partition_info.__isset.writer_assignment) { + return Status::InvalidArgument("External sink writer assignment is missing"); + } + if (_partition_info.writer_assignment != TExternalTableSinkWriterAssignment::IDENTITY && + _partition_info.writer_assignment != TExternalTableSinkWriterAssignment::SKEWED) { + return Status::InvalidArgument("Unsupported external sink writer assignment {}", + static_cast(_partition_info.writer_assignment)); + } + RETURN_IF_ERROR(logical_partition_count(_partition_count, _partition_info.writer_assignment, + &_logical_partition_count)); + const bool requires_identity = + _partition_info.algorithm == TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET; + if (requires_identity && + _partition_info.writer_assignment != TExternalTableSinkWriterAssignment::IDENTITY) { + return Status::InvalidArgument("Paimon bucket routing requires identity writer assignment"); + } + return create_external_partition_function(_partition_info, _logical_partition_count, + _hash_method, texprs, &_partition_function); +} + +Status ExternalTableSinkHashPartitioner::prepare(RuntimeState* state, + const RowDescriptor& row_desc) { + return _partition_function->prepare(state, row_desc); +} + +Status ExternalTableSinkHashPartitioner::open(RuntimeState* state) { + RETURN_IF_ERROR(_partition_function->open(state)); + + if (_partition_info.writer_assignment == TExternalTableSinkWriterAssignment::IDENTITY) { + _writer_assigner = std::make_unique(_partition_count); + } else { + const int task_num = state == nullptr ? 0 : state->task_num(); + _writer_assigner = std::make_unique( + cast_set(_logical_partition_count), cast_set(_partition_count), 1, + scale_writer_threshold_by_task( + config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold, + task_num), + scale_writer_threshold_by_task( + config::table_sink_partition_write_min_data_processed_rebalance_threshold, + task_num)); + } + return Status::OK(); +} + +Status ExternalTableSinkHashPartitioner::close(RuntimeState* state) { + return _partition_function->close(state); +} + +Status ExternalTableSinkHashPartitioner::do_partitioning(RuntimeState* state, Block* block) const { + if (_writer_assigner == nullptr) { + return Status::InternalError("External sink writer assigner is not open"); + } + const size_t rows = block->rows(); + const size_t block_bytes = block->bytes(); + if (rows == 0) { + _logical_partition_ids.clear(); + _channel_ids.clear(); + return Status::OK(); + } + RETURN_IF_ERROR(_partition_function->get_partitions(state, block, _logical_partition_count, + _logical_partition_ids)); + + return _writer_assigner->assign(_logical_partition_ids, nullptr, rows, block_bytes, + _channel_ids); +} + +const std::vector& +ExternalTableSinkHashPartitioner::get_channel_ids() const { + return _channel_ids; +} + +Status ExternalTableSinkHashPartitioner::clone(RuntimeState* state, + std::unique_ptr& partitioner) { + auto cloned = std::make_unique(_partition_count, _hash_method, + _partition_info); + RETURN_IF_ERROR(_partition_function->clone(state, cloned->_partition_function)); + partitioner = std::move(cloned); + return Status::OK(); +} + +#include "common/compile_check_end.h" +} // namespace doris diff --git a/be/src/exec/partitioner/external/external_table_sink_hash_partitioner.h b/be/src/exec/partitioner/external/external_table_sink_hash_partitioner.h new file mode 100644 index 00000000000000..2f47a1005b464f --- /dev/null +++ b/be/src/exec/partitioner/external/external_table_sink_hash_partitioner.h @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "exec/partitioner/partitioner.h" +#include "exec/partitioner/writer_assigner.h" + +namespace doris { +#include "common/compile_check_begin.h" + +// Computes external sink logical partitions and maps them to Doris exchange channels. +// Optional partition transforms are evaluated transiently and never appended to the sink row. +class ExternalTableSinkHashPartitioner final : public PartitionerBase { +public: + ExternalTableSinkHashPartitioner(HashValType partition_count, ShuffleHashMethod hash_method, + TExternalTableSinkHashPartitionInfo partition_info); + + Status init(const std::vector& texprs) override; + Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override; + Status open(RuntimeState* state) override; + Status close(RuntimeState* state) override; + Status do_partitioning(RuntimeState* state, Block* block) const override; + const std::vector& get_channel_ids() const override; + Status clone(RuntimeState* state, std::unique_ptr& partitioner) override; + +private: + ShuffleHashMethod _hash_method; + TExternalTableSinkHashPartitionInfo _partition_info; + HashValType _logical_partition_count; + std::unique_ptr _partition_function; + mutable std::unique_ptr _writer_assigner; + mutable std::vector _logical_partition_ids; + mutable std::vector _channel_ids; +}; + +#include "common/compile_check_end.h" +} // namespace doris diff --git a/be/src/exec/partitioner/external/paimon_fixed_bucket_partition_function.cpp b/be/src/exec/partitioner/external/paimon_fixed_bucket_partition_function.cpp new file mode 100644 index 00000000000000..59b5a382936cb2 --- /dev/null +++ b/be/src/exec/partitioner/external/paimon_fixed_bucket_partition_function.cpp @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/partitioner/external/paimon_fixed_bucket_partition_function.h" + +#include "common/status.h" +#include "exec/partitioner/external/paimon_native_row_hash.h" + +namespace doris { +#include "common/compile_check_begin.h" + +PaimonFixedBucketPartitionFunction::PaimonFixedBucketPartitionFunction( + HashValType partition_count, TPaimonFixedBucketInfo fixed_bucket_info) + : PaimonRowHashPartitionFunction(partition_count), + _fixed_bucket_info(std::move(fixed_bucket_info)) {} + +Status PaimonFixedBucketPartitionFunction::init(const std::vector& texprs) { + RETURN_IF_ERROR(PaimonRowHashPartitionFunction::init(texprs)); + if (_fixed_bucket_info.num_buckets <= 0) { + return Status::InvalidArgument("Paimon fixed-bucket count must be positive"); + } + RETURN_IF_ERROR(_validate_field_indexes(_fixed_bucket_info.partition_field_indexes, false)); + return _validate_field_indexes(_fixed_bucket_info.bucket_field_indexes, true); +} + +Status PaimonFixedBucketPartitionFunction::get_partitions( + RuntimeState* /*state*/, Block* block, size_t partition_count, + std::vector& partitions) const { + if (partition_count != _partition_count) { + return Status::InvalidArgument("Paimon writer count {} does not match planned count {}", + partition_count, _partition_count); + } + const size_t rows = block->rows(); + if (rows == 0) { + partitions.clear(); + return Status::OK(); + } + + std::vector fields; + RETURN_IF_ERROR(_evaluate_fields(block, fields)); + std::vector partition_hashes; + std::vector bucket_hashes; + RETURN_IF_ERROR( + _hash_fields(_fixed_bucket_info.partition_field_indexes, fields, partition_hashes)); + RETURN_IF_ERROR(_hash_fields(_fixed_bucket_info.bucket_field_indexes, fields, bucket_hashes)); + partitions.resize(rows); + for (size_t row = 0; row < rows; ++row) { + auto bucket = + paimon_native::default_bucket(bucket_hashes[row], _fixed_bucket_info.num_buckets); + if (!bucket.has_value()) { + return Status::InternalError("Failed to compute Paimon fixed bucket"); + } + auto channel = paimon_native::fixed_bucket_channel(partition_hashes[row], *bucket, + _partition_count); + if (!channel.has_value()) { + return Status::InternalError("Failed to compute Paimon fixed-bucket writer"); + } + partitions[row] = *channel; + } + return Status::OK(); +} + +Status PaimonFixedBucketPartitionFunction::clone( + RuntimeState* state, std::unique_ptr& function) const { + auto cloned = std::make_unique(_partition_count, + _fixed_bucket_info); + RETURN_IF_ERROR(_clone_expr_ctxs(state, cloned->_field_expr_ctxs)); + function = std::move(cloned); + return Status::OK(); +} + +#include "common/compile_check_end.h" +} // namespace doris diff --git a/be/src/exec/partitioner/external/paimon_fixed_bucket_partition_function.h b/be/src/exec/partitioner/external/paimon_fixed_bucket_partition_function.h new file mode 100644 index 00000000000000..e8b71ecf7d5ed5 --- /dev/null +++ b/be/src/exec/partitioner/external/paimon_fixed_bucket_partition_function.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "exec/partitioner/external/paimon_row_hash_partition_function.h" + +namespace doris { +#include "common/compile_check_begin.h" + +// Stateless native implementation of Paimon FixedBucketWriteSelector for the +// explicitly supported primitive routing types. +class PaimonFixedBucketPartitionFunction final : public PaimonRowHashPartitionFunction { +public: + PaimonFixedBucketPartitionFunction(HashValType partition_count, + TPaimonFixedBucketInfo fixed_bucket_info); + + Status init(const std::vector& texprs) override; + Status get_partitions(RuntimeState* state, Block* block, size_t partition_count, + std::vector& partitions) const override; + Status clone(RuntimeState* state, std::unique_ptr& function) const override; + +private: + TPaimonFixedBucketInfo _fixed_bucket_info; +}; + +#include "common/compile_check_end.h" +} // namespace doris diff --git a/be/src/exec/partitioner/external/paimon_native_row_hash.cpp b/be/src/exec/partitioner/external/paimon_native_row_hash.cpp new file mode 100644 index 00000000000000..a702fdfc219a38 --- /dev/null +++ b/be/src/exec/partitioner/external/paimon_native_row_hash.cpp @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/partitioner/external/paimon_native_row_hash.h" + +#include +#include +#include +#include + +namespace doris::paimon_native { +namespace { + +constexpr uint32_t MURMUR_C1 = 0xcc9e2d51U; +constexpr uint32_t MURMUR_C2 = 0x1b873593U; +constexpr uint32_t MURMUR_SEED = 42U; + +uint32_t rotate_left(uint32_t value, int distance) { + return (value << distance) | (value >> (32 - distance)); +} + +uint32_t mix_k1(uint32_t value) { + value *= MURMUR_C1; + value = rotate_left(value, 15); + value *= MURMUR_C2; + return value; +} + +uint32_t mix_h1(uint32_t hash, uint32_t value) { + hash ^= value; + hash = rotate_left(hash, 13); + return hash * 5U + 0xe6546b64U; +} + +uint32_t fmix(uint32_t hash, size_t length) { + hash ^= static_cast(length); + hash ^= hash >> 16; + hash *= 0x85ebca6bU; + hash ^= hash >> 13; + hash *= 0xc2b2ae35U; + hash ^= hash >> 16; + return hash; +} + +template +void put_native(std::vector* bytes, size_t offset, T value) { + std::memcpy(bytes->data() + offset, &value, sizeof(value)); +} + +size_t round_to_word(size_t size) { + return (size + 7U) & ~size_t {7U}; +} + +} // namespace + +BinaryRowEncoder::BinaryRowEncoder(size_t arity) + : _arity(arity), + _null_bits_size((arity + 63U + 8U) / 64U * 8U), + _fixed_size(_null_bits_size + arity * 8U), + _cursor(_fixed_size), + _bytes(_fixed_size, 0) {} + +void BinaryRowEncoder::reset() { + _cursor = _fixed_size; + _bytes.assign(_fixed_size, 0); +} + +bool BinaryRowEncoder::_valid_position(size_t position) const { + return position < _arity; +} + +size_t BinaryRowEncoder::_field_offset(size_t position) const { + return _null_bits_size + position * 8U; +} + +void BinaryRowEncoder::_set_null_bit(size_t position) { + const size_t bit_index = position + 8U; + _bytes[bit_index >> 3U] |= static_cast(1U << (bit_index & 7U)); +} + +void BinaryRowEncoder::_ensure_capacity(size_t size) { + if (_bytes.size() < size) { + _bytes.resize(size, 0); + } +} + +void BinaryRowEncoder::_set_offset_and_size(size_t position, uint32_t offset, uint32_t size) { + uint64_t offset_and_size = (static_cast(offset) << 32U) | size; + put_native(&_bytes, _field_offset(position), offset_and_size); +} + +bool BinaryRowEncoder::set_null(size_t position) { + if (!_valid_position(position)) { + return false; + } + _set_null_bit(position); + std::fill_n(_bytes.begin() + _field_offset(position), 8, uint8_t {0}); + return true; +} + +bool BinaryRowEncoder::write_boolean(size_t position, bool value) { + return write_tinyint(position, value ? 1 : 0); +} + +bool BinaryRowEncoder::write_tinyint(size_t position, int8_t value) { + if (!_valid_position(position)) { + return false; + } + put_native(&_bytes, _field_offset(position), value); + return true; +} + +bool BinaryRowEncoder::write_smallint(size_t position, int16_t value) { + if (!_valid_position(position)) { + return false; + } + put_native(&_bytes, _field_offset(position), value); + return true; +} + +bool BinaryRowEncoder::write_int(size_t position, int32_t value) { + if (!_valid_position(position)) { + return false; + } + put_native(&_bytes, _field_offset(position), value); + return true; +} + +bool BinaryRowEncoder::write_bigint(size_t position, int64_t value) { + if (!_valid_position(position)) { + return false; + } + put_native(&_bytes, _field_offset(position), value); + return true; +} + +bool BinaryRowEncoder::write_float(size_t position, float value) { + if (!_valid_position(position)) { + return false; + } + put_native(&_bytes, _field_offset(position), value); + return true; +} + +bool BinaryRowEncoder::write_double(size_t position, double value) { + if (!_valid_position(position)) { + return false; + } + put_native(&_bytes, _field_offset(position), value); + return true; +} + +bool BinaryRowEncoder::_write_bytes(size_t position, std::string_view bytes) { + if (!_valid_position(position) || bytes.size() > std::numeric_limits::max()) { + return false; + } + if (bytes.size() <= 7U) { + uint64_t inline_value = (static_cast(bytes.size()) | 0x80U) << 56U; + for (size_t index = 0; index < bytes.size(); ++index) { + if constexpr (std::endian::native == std::endian::little) { + inline_value |= static_cast(static_cast(bytes[index])) + << (index * 8U); + } else { + inline_value |= static_cast(static_cast(bytes[index])) + << ((6U - index) * 8U); + } + } + put_native(&_bytes, _field_offset(position), inline_value); + return true; + } + + const size_t rounded_size = round_to_word(bytes.size()); + if (rounded_size > std::numeric_limits::max() || + _cursor > std::numeric_limits::max() - rounded_size) { + return false; + } + _ensure_capacity(_cursor + rounded_size); + std::fill_n(_bytes.begin() + _cursor, rounded_size, uint8_t {0}); + std::memcpy(_bytes.data() + _cursor, bytes.data(), bytes.size()); + _set_offset_and_size(position, static_cast(_cursor), + static_cast(bytes.size())); + _cursor += rounded_size; + return true; +} + +bool BinaryRowEncoder::write_string(size_t position, std::string_view utf8) { + return _write_bytes(position, utf8); +} + +bool BinaryRowEncoder::write_binary(size_t position, std::string_view bytes) { + return _write_bytes(position, bytes); +} + +int32_t BinaryRowEncoder::hash() const { + return binary_row_hash(std::string_view(reinterpret_cast(_bytes.data()), _cursor)); +} + +int32_t binary_row_hash(std::string_view bytes) { + uint32_t hash = MURMUR_SEED; + size_t offset = 0; + const size_t aligned_length = bytes.size() - bytes.size() % 4U; + for (; offset < aligned_length; offset += 4U) { + uint32_t word; + std::memcpy(&word, bytes.data() + offset, sizeof(word)); + hash = mix_h1(hash, mix_k1(word)); + } + for (; offset < bytes.size(); ++offset) { + int32_t signed_byte = static_cast(bytes[offset]); + hash = mix_h1(hash, mix_k1(static_cast(signed_byte))); + } + uint32_t mixed = fmix(hash, bytes.size()); + int32_t result; + std::memcpy(&result, &mixed, sizeof(result)); + return result; +} + +std::optional default_bucket(int32_t bucket_key_hash, int32_t num_buckets) { + if (num_buckets <= 0) { + return std::nullopt; + } + int32_t remainder = bucket_key_hash % num_buckets; + return static_cast(remainder < 0 ? -remainder : remainder); +} + +std::optional fixed_bucket_channel(int32_t partition_hash, uint32_t bucket, + uint32_t num_channels) { + if (num_channels == 0) { + return std::nullopt; + } + if (partition_hash == std::numeric_limits::min()) { + partition_hash = std::numeric_limits::max(); + } + uint32_t start_channel = + static_cast(partition_hash < 0 ? -partition_hash : partition_hash) % + num_channels; + return static_cast((static_cast(start_channel) + bucket) % num_channels); +} + +} // namespace doris::paimon_native diff --git a/be/src/exec/partitioner/external/paimon_native_row_hash.h b/be/src/exec/partitioner/external/paimon_native_row_hash.h new file mode 100644 index 00000000000000..49f3a1a924e6f4 --- /dev/null +++ b/be/src/exec/partitioner/external/paimon_native_row_hash.h @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace doris::paimon_native { + +// Builds the byte representation used by Paimon BinaryRowWriter for the supported routing +// types. MemorySegment primitive access is native-endian. +class BinaryRowEncoder { +public: + explicit BinaryRowEncoder(size_t arity); + + void reset(); + bool set_null(size_t position); + bool write_boolean(size_t position, bool value); + bool write_tinyint(size_t position, int8_t value); + bool write_smallint(size_t position, int16_t value); + bool write_int(size_t position, int32_t value); + bool write_bigint(size_t position, int64_t value); + bool write_float(size_t position, float value); + bool write_double(size_t position, double value); + bool write_string(size_t position, std::string_view utf8); + bool write_binary(size_t position, std::string_view bytes); + + int32_t hash() const; + const std::vector& bytes() const { return _bytes; } + +private: + bool _valid_position(size_t position) const; + size_t _field_offset(size_t position) const; + void _set_null_bit(size_t position); + void _ensure_capacity(size_t size); + void _set_offset_and_size(size_t position, uint32_t offset, uint32_t size); + bool _write_bytes(size_t position, std::string_view bytes); + + const size_t _arity; + const size_t _null_bits_size; + const size_t _fixed_size; + size_t _cursor; + std::vector _bytes; +}; + +int32_t binary_row_hash(std::string_view bytes); + +// Reproduces Paimon DefaultBucketFunction. nullopt means invalid metadata. +std::optional default_bucket(int32_t bucket_key_hash, int32_t num_buckets); + +// Reproduces ChannelComputer.select(partition, bucket, numChannels). +std::optional fixed_bucket_channel(int32_t partition_hash, uint32_t bucket, + uint32_t num_channels); + +} // namespace doris::paimon_native diff --git a/be/src/exec/partitioner/external/paimon_row_hash_partition_function.cpp b/be/src/exec/partitioner/external/paimon_row_hash_partition_function.cpp new file mode 100644 index 00000000000000..f1ddb17c9d5a5a --- /dev/null +++ b/be/src/exec/partitioner/external/paimon_row_hash_partition_function.cpp @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/partitioner/external/paimon_row_hash_partition_function.h" + +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type_nullable.h" +#include "exec/partitioner/external/paimon_native_row_hash.h" + +namespace doris { +#include "common/compile_check_begin.h" + +namespace { +bool is_supported_type(PrimitiveType type) { + switch (type) { + case TYPE_BOOLEAN: + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_FLOAT: + case TYPE_DOUBLE: + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: + case TYPE_BINARY: + case TYPE_VARBINARY: + return true; + default: + return false; + } +} + +template +bool read_fixed_value(const IColumn& column, size_t row, T* value) { + StringRef data = column.get_data_at(row); + if (data.size != sizeof(T)) { + return false; + } + std::memcpy(value, data.data, sizeof(T)); + return true; +} + +Status encode_field(paimon_native::BinaryRowEncoder* encoder, size_t target_position, + const ColumnWithTypeAndName& field, size_t row) { + const IColumn& column = *field.column; + if (column.is_null_at(row)) { + if (!encoder->set_null(target_position)) { + return Status::InternalError("Failed to encode null Paimon routing field {}", + target_position); + } + return Status::OK(); + } + + bool encoded = false; + switch (remove_nullable(field.type)->get_primitive_type()) { + case TYPE_BOOLEAN: { + uint8_t value = 0; + encoded = read_fixed_value(column, row, &value) && + encoder->write_boolean(target_position, value != 0); + break; + } + case TYPE_TINYINT: { + int8_t value = 0; + encoded = read_fixed_value(column, row, &value) && + encoder->write_tinyint(target_position, value); + break; + } + case TYPE_SMALLINT: { + int16_t value = 0; + encoded = read_fixed_value(column, row, &value) && + encoder->write_smallint(target_position, value); + break; + } + case TYPE_INT: { + int32_t value = 0; + encoded = + read_fixed_value(column, row, &value) && encoder->write_int(target_position, value); + break; + } + case TYPE_BIGINT: { + int64_t value = 0; + encoded = read_fixed_value(column, row, &value) && + encoder->write_bigint(target_position, value); + break; + } + case TYPE_FLOAT: { + float value = 0; + encoded = read_fixed_value(column, row, &value) && + encoder->write_float(target_position, value); + break; + } + case TYPE_DOUBLE: { + double value = 0; + encoded = read_fixed_value(column, row, &value) && + encoder->write_double(target_position, value); + break; + } + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: { + StringRef value = column.get_data_at(row); + encoded = encoder->write_string(target_position, std::string_view(value.data, value.size)); + break; + } + case TYPE_BINARY: + case TYPE_VARBINARY: { + StringRef value = column.get_data_at(row); + encoded = encoder->write_binary(target_position, std::string_view(value.data, value.size)); + break; + } + default: + return Status::InvalidArgument("Unsupported Doris type {} for Paimon native routing", + field.type->get_name()); + } + if (!encoded) { + return Status::InvalidArgument("Doris column {} cannot be encoded for Paimon routing", + field.name); + } + return Status::OK(); +} + +Status encode_fields(paimon_native::BinaryRowEncoder* encoder, + const std::vector& field_indexes, + const std::vector& fields, size_t row) { + encoder->reset(); + for (size_t position = 0; position < field_indexes.size(); ++position) { + RETURN_IF_ERROR(encode_field(encoder, position, fields[field_indexes[position]], row)); + } + return Status::OK(); +} +} // namespace + +PaimonRowHashPartitionFunction::PaimonRowHashPartitionFunction(HashValType partition_count) + : _partition_count(partition_count) {} + +Status PaimonRowHashPartitionFunction::init(const std::vector& texprs) { + if (_partition_count == 0) { + return Status::InvalidArgument("Paimon writer count must be positive"); + } + RETURN_IF_ERROR(VExpr::create_expr_trees(texprs, _field_expr_ctxs)); + for (const auto& context : _field_expr_ctxs) { + PrimitiveType type = remove_nullable(context->root()->data_type())->get_primitive_type(); + if (!is_supported_type(type)) { + return Status::InvalidArgument("Unsupported Paimon native routing type {}", + context->root()->data_type()->get_name()); + } + } + return Status::OK(); +} + +Status PaimonRowHashPartitionFunction::_validate_field_indexes(const std::vector& indexes, + bool require_non_empty) const { + if (require_non_empty && indexes.empty()) { + return Status::InvalidArgument("Paimon routing fields are missing"); + } + for (int32_t index : indexes) { + if (index < 0 || index >= _field_expr_ctxs.size()) { + return Status::InvalidArgument("Invalid Paimon routing field index {}", index); + } + } + return Status::OK(); +} + +Status PaimonRowHashPartitionFunction::prepare(RuntimeState* state, const RowDescriptor& row_desc) { + return VExpr::prepare(_field_expr_ctxs, state, row_desc); +} + +Status PaimonRowHashPartitionFunction::open(RuntimeState* state) { + return VExpr::open(_field_expr_ctxs, state); +} + +Status PaimonRowHashPartitionFunction::_evaluate_fields( + Block* block, std::vector& fields) const { + fields.resize(_field_expr_ctxs.size()); + for (size_t index = 0; index < _field_expr_ctxs.size(); ++index) { + RETURN_IF_ERROR(_field_expr_ctxs[index]->execute(block, fields[index])); + } + return Status::OK(); +} + +Status PaimonRowHashPartitionFunction::_hash_fields( + const std::vector& indexes, const std::vector& fields, + std::vector& hashes) const { + paimon_native::BinaryRowEncoder encoder(indexes.size()); + hashes.resize(fields.empty() ? 0 : fields.front().column->size()); + for (size_t row = 0; row < hashes.size(); ++row) { + RETURN_IF_ERROR(encode_fields(&encoder, indexes, fields, row)); + hashes[row] = encoder.hash(); + } + return Status::OK(); +} + +Status PaimonRowHashPartitionFunction::_clone_expr_ctxs(RuntimeState* state, + VExprContextSPtrs& destination) const { + destination.resize(_field_expr_ctxs.size()); + for (size_t index = 0; index < _field_expr_ctxs.size(); ++index) { + RETURN_IF_ERROR(_field_expr_ctxs[index]->clone(state, destination[index])); + } + return Status::OK(); +} + +#include "common/compile_check_end.h" +} // namespace doris diff --git a/be/src/exec/partitioner/external/paimon_row_hash_partition_function.h b/be/src/exec/partitioner/external/paimon_row_hash_partition_function.h new file mode 100644 index 00000000000000..6aaa8e2bd1a11e --- /dev/null +++ b/be/src/exec/partitioner/external/paimon_row_hash_partition_function.h @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "exec/partitioner/partitioner.h" + +namespace doris { +#include "common/compile_check_begin.h" + +// Shared expression lifecycle and BinaryRow hashing for Paimon routing functions. +class PaimonRowHashPartitionFunction : public PartitionFunction { +public: + explicit PaimonRowHashPartitionFunction(HashValType partition_count); + + Status init(const std::vector& texprs) override; + Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override; + Status open(RuntimeState* state) override; + Status close(RuntimeState* state) override { return Status::OK(); } + HashValType partition_count() const override { return _partition_count; } + +protected: + Status _validate_field_indexes(const std::vector& indexes, + bool require_non_empty) const; + Status _evaluate_fields(Block* block, std::vector& fields) const; + Status _hash_fields(const std::vector& indexes, + const std::vector& fields, + std::vector& hashes) const; + Status _clone_expr_ctxs(RuntimeState* state, VExprContextSPtrs& destination) const; + + const HashValType _partition_count; + VExprContextSPtrs _field_expr_ctxs; +}; + +#include "common/compile_check_end.h" +} // namespace doris diff --git a/be/src/exec/partitioner/partitioner.cpp b/be/src/exec/partitioner/partitioner.cpp index bd0c9ba7709df5..f8cc7555ca628b 100644 --- a/be/src/exec/partitioner/partitioner.cpp +++ b/be/src/exec/partitioner/partitioner.cpp @@ -85,6 +85,51 @@ Status Crc32CHashPartitioner::clone(RuntimeState* state, return _clone_expr_ctxs(state, new_partitioner->_partition_expr_ctxs); } +HashPartitionFunction::HashPartitionFunction(HashValType partition_count, + ShuffleHashMethod hash_method) + : _partition_count(partition_count), _hash_method(hash_method) {} + +Status HashPartitionFunction::init(const std::vector& texprs) { + if (_hash_method == ShuffleHashMethod::CRC32C) { + _partitioner = std::make_unique(_partition_count); + } else { + _partitioner = std::make_unique>(_partition_count); + } + return _partitioner->init(texprs); +} + +Status HashPartitionFunction::prepare(RuntimeState* state, const RowDescriptor& row_desc) { + return _partitioner->prepare(state, row_desc); +} + +Status HashPartitionFunction::open(RuntimeState* state) { + return _partitioner->open(state); +} + +Status HashPartitionFunction::close(RuntimeState* state) { + return _partitioner->close(state); +} + +Status HashPartitionFunction::get_partitions(RuntimeState* state, Block* block, + size_t partition_count, + std::vector& partitions) const { + if (partition_count != _partition_count) { + return Status::InvalidArgument("Hash partition count {} does not match planned count {}", + partition_count, _partition_count); + } + RETURN_IF_ERROR(_partitioner->do_partitioning(state, block)); + partitions = _partitioner->get_channel_ids(); + return Status::OK(); +} + +Status HashPartitionFunction::clone(RuntimeState* state, + std::unique_ptr& function) const { + auto cloned = std::make_unique(_partition_count, _hash_method); + RETURN_IF_ERROR(_partitioner->clone(state, cloned->_partitioner)); + function = std::move(cloned); + return Status::OK(); +} + template class Crc32HashPartitioner; template class Crc32HashPartitioner; template class Crc32HashPartitioner; diff --git a/be/src/exec/partitioner/partitioner.h b/be/src/exec/partitioner/partitioner.h index 15c2b578ef88ff..a05569ba82b762 100644 --- a/be/src/exec/partitioner/partitioner.h +++ b/be/src/exec/partitioner/partitioner.h @@ -56,6 +56,11 @@ class PartitionerBase { const HashValType _partition_count; }; +enum class ShuffleHashMethod { + CRC32, + CRC32C, +}; + class PartitionFunction { public: using HashValType = PartitionerBase::HashValType; @@ -79,9 +84,25 @@ class PartitionFunction { std::unique_ptr& function) const = 0; }; -enum class ShuffleHashMethod { - CRC32, - CRC32C, +// Adapts the standard Doris expression hash partitioner to the composable +// PartitionFunction interface used by sink routing. +class HashPartitionFunction final : public PartitionFunction { +public: + HashPartitionFunction(HashValType partition_count, ShuffleHashMethod hash_method); + + Status init(const std::vector& texprs) override; + Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override; + Status open(RuntimeState* state) override; + Status close(RuntimeState* state) override; + Status get_partitions(RuntimeState* state, Block* block, size_t partition_count, + std::vector& partitions) const override; + HashValType partition_count() const override { return _partition_count; } + Status clone(RuntimeState* state, std::unique_ptr& function) const override; + +private: + HashValType _partition_count; + ShuffleHashMethod _hash_method; + std::unique_ptr _partitioner; }; template diff --git a/be/src/exec/partitioner/writer_assigner.cpp b/be/src/exec/partitioner/writer_assigner.cpp new file mode 100644 index 00000000000000..b0628c397734b3 --- /dev/null +++ b/be/src/exec/partitioner/writer_assigner.cpp @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/partitioner/writer_assigner.h" + +#include "exec/connector/skewed_partition_rebalancer.h" + +namespace doris { +#include "common/compile_check_begin.h" + +namespace { +Status validate_assignment_input(const std::vector& partition_ids, + const std::vector* mask, size_t rows) { + if (partition_ids.size() < rows) { + return Status::InvalidArgument("Writer assignment has {} partition ids for {} rows", + partition_ids.size(), rows); + } + if (mask != nullptr && mask->size() < rows) { + return Status::InvalidArgument("Writer assignment mask has {} entries for {} rows", + mask->size(), rows); + } + return Status::OK(); +} +} // namespace + +Status IdentityWriterAssigner::assign(const std::vector& partition_ids, + const std::vector* mask, size_t rows, + size_t /*block_bytes*/, std::vector& writer_ids) { + RETURN_IF_ERROR(validate_assignment_input(partition_ids, mask, rows)); + if (writer_ids.size() != rows && &writer_ids != &partition_ids) { + writer_ids.resize(rows); + } + for (size_t row = 0; row < rows; ++row) { + if (mask != nullptr && (*mask)[row] == 0) { + continue; + } + if (partition_ids[row] >= _writer_count) { + return Status::InvalidArgument("Logical partition {} exceeds writer count {}", + partition_ids[row], _writer_count); + } + writer_ids[row] = partition_ids[row]; + } + return Status::OK(); +} + +SkewedWriterAssigner::SkewedWriterAssigner(int partition_count, int task_count, + int task_bucket_count, + long min_partition_data_processed_rebalance_threshold, + long min_data_processed_rebalance_threshold) + : _rebalancer(std::make_unique( + partition_count, task_count, task_bucket_count, + min_partition_data_processed_rebalance_threshold, + min_data_processed_rebalance_threshold)), + _writer_count(task_count), + _partition_row_counts(partition_count, 0), + _partition_writer_ids(partition_count, -1), + _partition_writer_indexes(partition_count, 0) {} + +SkewedWriterAssigner::~SkewedWriterAssigner() = default; + +Status SkewedWriterAssigner::assign(const std::vector& partition_ids, + const std::vector* mask, size_t rows, + size_t block_bytes, std::vector& writer_ids) { + RETURN_IF_ERROR(validate_assignment_input(partition_ids, mask, rows)); + if (rows == 0) { + return Status::OK(); + } + if (_partition_row_counts.empty()) { + return Status::InvalidArgument("Skewed writer assignment has no logical partitions"); + } + if (writer_ids.size() != rows && &writer_ids != &partition_ids) { + writer_ids.resize(rows); + } + + std::fill(_partition_row_counts.begin(), _partition_row_counts.end(), 0); + std::fill(_partition_writer_ids.begin(), _partition_writer_ids.end(), -1); + _rebalancer->rebalance(); + + const size_t partition_count = _partition_row_counts.size(); + for (size_t row = 0; row < rows; ++row) { + if (mask != nullptr && (*mask)[row] == 0) { + continue; + } + const uint32_t partition_id = partition_ids[row]; + if (partition_id >= partition_count) { + return Status::InvalidArgument("Logical partition {} exceeds partition count {}", + partition_id, partition_count); + } + _partition_row_counts[partition_id] += 1; + int writer_id = _partition_writer_ids[partition_id]; + if (writer_id == -1) { + writer_id = _get_next_writer_id(partition_id); + if (writer_id < 0 || writer_id >= _writer_count) { + return Status::InternalError("Skewed writer assignment returned invalid writer {}", + writer_id); + } + _partition_writer_ids[partition_id] = writer_id; + } + writer_ids[row] = static_cast(writer_id); + } + + for (size_t partition_id = 0; partition_id < partition_count; ++partition_id) { + if (_partition_row_counts[partition_id] > 0) { + _rebalancer->add_partition_row_count(static_cast(partition_id), + _partition_row_counts[partition_id]); + } + } + _rebalancer->add_data_processed(static_cast(block_bytes)); + return Status::OK(); +} + +int SkewedWriterAssigner::_get_next_writer_id(uint32_t partition_id) { + return _rebalancer->get_task_id(partition_id, _partition_writer_indexes[partition_id]++); +} + +int64_t scale_writer_threshold_by_task(int64_t value, int task_num) { + if (task_num <= 0) { + return value; + } + int64_t scaled = value / task_num; + return scaled == 0 ? value : scaled; +} + +#include "common/compile_check_end.h" +} // namespace doris diff --git a/be/src/exec/partitioner/writer_assigner.h b/be/src/exec/partitioner/writer_assigner.h new file mode 100644 index 00000000000000..7657c4c4c76ed9 --- /dev/null +++ b/be/src/exec/partitioner/writer_assigner.h @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris { +class SkewedPartitionRebalancer; +} + +namespace doris { +#include "common/compile_check_begin.h" + +// Maps logical partitions computed by a PartitionFunction to Doris exchange channels. +class WriterAssigner { +public: + virtual ~WriterAssigner() = default; + + virtual Status assign(const std::vector& partition_ids, + const std::vector* mask, size_t rows, size_t block_bytes, + std::vector& writer_ids) = 0; +}; + +// Preserves stable ownership: one logical partition always maps to one writer id. +class IdentityWriterAssigner final : public WriterAssigner { +public: + explicit IdentityWriterAssigner(uint32_t writer_count) : _writer_count(writer_count) {} + + Status assign(const std::vector& partition_ids, const std::vector* mask, + size_t rows, size_t block_bytes, std::vector& writer_ids) override; + +private: + uint32_t _writer_count; +}; + +// Allows a hot logical partition to use multiple writers while retaining the existing +// ScaleWriter affinity and rebalance behavior. +class SkewedWriterAssigner final : public WriterAssigner { +public: + SkewedWriterAssigner(int partition_count, int task_count, int task_bucket_count, + long min_partition_data_processed_rebalance_threshold, + long min_data_processed_rebalance_threshold); + + ~SkewedWriterAssigner() override; + + Status assign(const std::vector& partition_ids, const std::vector* mask, + size_t rows, size_t block_bytes, std::vector& writer_ids) override; + +private: + int _get_next_writer_id(uint32_t partition_id); + + std::unique_ptr _rebalancer; + int _writer_count; + std::vector _partition_row_counts; + std::vector _partition_writer_ids; + std::vector _partition_writer_indexes; +}; + +// Scale table-sink thresholds by local pipeline task count while preserving the historical +// behavior for very small values. +int64_t scale_writer_threshold_by_task(int64_t value, int task_num); + +#include "common/compile_check_end.h" +} // namespace doris diff --git a/be/src/exec/sink/scale_writer_partitioning_exchanger.hpp b/be/src/exec/sink/scale_writer_partitioning_exchanger.hpp deleted file mode 100644 index 3756dbe11e7898..00000000000000 --- a/be/src/exec/sink/scale_writer_partitioning_exchanger.hpp +++ /dev/null @@ -1,128 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "core/block/block.h" -#include "exec/connector/skewed_partition_rebalancer.h" -#include "exec/partitioner/partitioner.h" - -namespace doris { -#include "common/compile_check_begin.h" -class ScaleWriterPartitioner final : public PartitionerBase { -public: - ScaleWriterPartitioner(int channel_size, int partition_count, int task_count, - int task_bucket_count, - long min_partition_data_processed_rebalance_threshold, - long min_data_processed_rebalance_threshold) - : PartitionerBase(partition_count), - _channel_size(channel_size), - _partition_rebalancer(partition_count, task_count, task_bucket_count, - min_partition_data_processed_rebalance_threshold, - min_data_processed_rebalance_threshold), - _partition_row_counts(partition_count, 0), - _partition_writer_ids(partition_count, -1), - _partition_writer_indexes(partition_count, 0), - _task_count(task_count), - _task_bucket_count(task_bucket_count), - _min_partition_data_processed_rebalance_threshold( - min_partition_data_processed_rebalance_threshold), - _min_data_processed_rebalance_threshold(min_data_processed_rebalance_threshold) { - _crc_partitioner = - std::make_unique>(_partition_count); - } - - ~ScaleWriterPartitioner() override = default; - - Status init(const std::vector& texprs) override { - return _crc_partitioner->init(texprs); - } - - Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override { - return _crc_partitioner->prepare(state, row_desc); - } - - Status open(RuntimeState* state) override { return _crc_partitioner->open(state); } - - Status close(RuntimeState* state) override { return _crc_partitioner->close(state); } - - Status do_partitioning(RuntimeState* state, Block* block) const override { - _hash_vals.resize(block->rows()); - for (int partition_id = 0; partition_id < _partition_row_counts.size(); partition_id++) { - _partition_row_counts[partition_id] = 0; - _partition_writer_ids[partition_id] = -1; - } - - _partition_rebalancer.rebalance(); - - RETURN_IF_ERROR(_crc_partitioner->do_partitioning(state, block)); - const auto& channel_ids = _crc_partitioner->get_channel_ids(); - for (size_t position = 0; position < block->rows(); position++) { - auto partition_id = channel_ids[position]; - _partition_row_counts[partition_id] += 1; - - // Get writer id for this partition by looking at the scaling state - int writer_id = _partition_writer_ids[partition_id]; - if (writer_id == -1) { - writer_id = _get_next_writer_id(partition_id); - _partition_writer_ids[partition_id] = writer_id; - } - _hash_vals[position] = writer_id; - } - - for (int partition_id = 0; partition_id < _partition_row_counts.size(); partition_id++) { - _partition_rebalancer.add_partition_row_count(partition_id, - _partition_row_counts[partition_id]); - } - _partition_rebalancer.add_data_processed(block->bytes()); - - return Status::OK(); - } - - const std::vector& get_channel_ids() const override { return _hash_vals; } - - Status clone(RuntimeState* state, std::unique_ptr& partitioner) override { - partitioner = std::make_unique( - _channel_size, (int)_partition_count, _task_count, _task_bucket_count, - _min_partition_data_processed_rebalance_threshold, - _min_data_processed_rebalance_threshold); - return Status::OK(); - } - -private: - int _get_next_writer_id(HashValType partition_id) const { - return _partition_rebalancer.get_task_id(partition_id, - _partition_writer_indexes[partition_id]++); - } - - int _channel_size; - std::unique_ptr _crc_partitioner; - mutable SkewedPartitionRebalancer _partition_rebalancer; - mutable std::vector _partition_row_counts; - mutable std::vector _partition_writer_ids; - mutable std::vector _partition_writer_indexes; - mutable std::vector _hash_vals; - const int _task_count; - const int _task_bucket_count; - const long _min_partition_data_processed_rebalance_threshold; - const long _min_data_processed_rebalance_threshold; -}; -#include "common/compile_check_end.h" -} // namespace doris diff --git a/be/src/format/transformer/iceberg_partition_function.cpp b/be/src/format/transformer/iceberg_partition_function.cpp index 2217b9833286cb..14eba85e28a88e 100644 --- a/be/src/format/transformer/iceberg_partition_function.cpp +++ b/be/src/format/transformer/iceberg_partition_function.cpp @@ -19,7 +19,6 @@ #include "common/cast_set.h" #include "common/exception.h" -#include "common/logging.h" #include "common/status.h" #include "core/column/column_const.h" #include "core/column/column_nullable.h" @@ -124,10 +123,8 @@ Status IcebergInsertPartitionFunction::open(RuntimeState* state) { field.transformer = PartitionColumnTransforms::create( partition_field, field.expr_ctx->root()->data_type()); } catch (const doris::Exception& e) { - LOG(WARNING) << "Merge partitioning fallback to RR: " << e.what(); - _fallback_to_random = true; - _partition_fields.clear(); - break; + return Status::NotSupported("Unsupported Iceberg partition transform: {}", + e.what()); } } } @@ -137,9 +134,6 @@ Status IcebergInsertPartitionFunction::open(RuntimeState* state) { Status IcebergInsertPartitionFunction::get_partitions(RuntimeState* /*state*/, Block* block, size_t partition_count, std::vector& partitions) const { - if (_fallback_to_random) { - return Status::InternalError("Merge partitioning fallback to random"); - } if (partition_count == 0) { return Status::InternalError("Partition count is zero"); } @@ -177,7 +171,6 @@ Status IcebergInsertPartitionFunction::clone(RuntimeState* state, new_function->_partition_fields.emplace_back(std::move(field)); } } - new_function->_fallback_to_random = _fallback_to_random; return Status::OK(); } diff --git a/be/src/format/transformer/iceberg_partition_function.h b/be/src/format/transformer/iceberg_partition_function.h index 216e313e9233e2..d7698ed78b4eff 100644 --- a/be/src/format/transformer/iceberg_partition_function.h +++ b/be/src/format/transformer/iceberg_partition_function.h @@ -44,8 +44,6 @@ class IcebergInsertPartitionFunction final : public PartitionFunction { HashValType partition_count() const override { return _partition_count; } Status clone(RuntimeState* state, std::unique_ptr& function) const override; - bool fallback_to_random() const { return _fallback_to_random; } - private: struct InsertPartitionField { std::string transform; @@ -66,7 +64,6 @@ class IcebergInsertPartitionFunction final : public PartitionFunction { std::vector _partition_fields_spec; VExprContextSPtrs _partition_expr_ctxs; std::vector _partition_fields; - bool _fallback_to_random = false; }; class IcebergDeletePartitionFunction final : public PartitionFunction { diff --git a/be/src/format/transformer/merge_partitioner.cpp b/be/src/format/transformer/merge_partitioner.cpp index 405dce01c6197d..3aabc88fc7f696 100644 --- a/be/src/format/transformer/merge_partitioner.cpp +++ b/be/src/format/transformer/merge_partitioner.cpp @@ -34,16 +34,6 @@ namespace doris { #include "common/compile_check_begin.h" -namespace { -int64_t scale_threshold_by_task(int64_t value, int task_num) { - if (task_num <= 0) { - return value; - } - int64_t scaled = value / task_num; - return scaled == 0 ? value : scaled; -} -} // namespace - MergePartitioner::MergePartitioner(size_t partition_count, const TMergePartitionInfo& merge_info, bool use_new_shuffle_hash_method) : PartitionerBase(static_cast(partition_count)), @@ -93,11 +83,12 @@ Status MergePartitioner::prepare(RuntimeState* state, const RowDescriptor& row_d Status MergePartitioner::open(RuntimeState* state) { RETURN_IF_ERROR(VExpr::open(_operation_expr_ctxs, state)); if (_insert_partition_function != nullptr) { - RETURN_IF_ERROR(_insert_partition_function->open(state)); - if (auto* insert_function = - dynamic_cast(_insert_partition_function.get()); - insert_function != nullptr && insert_function->fallback_to_random()) { + Status status = _insert_partition_function->open(state); + if (status.is()) { + LOG(WARNING) << "Merge partitioning fallback to RR: " << status; _insert_random = true; + } else { + RETURN_IF_ERROR(status); } } if (_delete_partition_function != nullptr) { @@ -184,7 +175,7 @@ Status MergePartitioner::do_partitioning(RuntimeState* state, Block* block) cons _insert_writer_count = static_cast(_partition_count); } } else if (_enable_insert_rebalance) { - _apply_insert_rebalance(ops, insert_hashes, block->bytes()); + RETURN_IF_ERROR(_apply_insert_rebalance(ops, insert_hashes, block->bytes())); } } @@ -277,14 +268,14 @@ Status MergePartitioner::clone(RuntimeState* state, std::unique_ptr& ops, - std::vector& insert_hashes, - size_t block_bytes) const { +Status MergePartitioner::_apply_insert_rebalance(const std::vector& ops, + std::vector& insert_hashes, + size_t block_bytes) const { if (!_enable_insert_rebalance || _insert_writer_assigner == nullptr) { - return; + return Status::OK(); } if (insert_hashes.empty() || _insert_partition_count == 0) { - return; + return Status::OK(); } std::vector mask(ops.size(), 0); for (size_t i = 0; i < ops.size(); ++i) { @@ -292,7 +283,8 @@ void MergePartitioner::_apply_insert_rebalance(const std::vector& ops, mask[i] = 1; } } - _insert_writer_assigner->assign(insert_hashes, &mask, ops.size(), block_bytes, insert_hashes); + return _insert_writer_assigner->assign(insert_hashes, &mask, ops.size(), block_bytes, + insert_hashes); } void MergePartitioner::_init_insert_scaling(RuntimeState* state) { @@ -325,10 +317,10 @@ void MergePartitioner::_init_insert_scaling(RuntimeState* state) { } int task_num = state == nullptr ? 0 : state->task_num(); - int64_t min_partition_threshold = scale_threshold_by_task( + int64_t min_partition_threshold = scale_writer_threshold_by_task( config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold, task_num); - int64_t min_data_threshold = scale_threshold_by_task( + int64_t min_data_threshold = scale_writer_threshold_by_task( config::table_sink_partition_write_min_data_processed_rebalance_threshold, task_num); _insert_writer_assigner = std::make_unique( diff --git a/be/src/format/transformer/merge_partitioner.h b/be/src/format/transformer/merge_partitioner.h index 9cb36d9a5c6e5f..41ffd62fd721e7 100644 --- a/be/src/format/transformer/merge_partitioner.h +++ b/be/src/format/transformer/merge_partitioner.h @@ -22,7 +22,7 @@ #include #include "exec/partitioner/partitioner.h" -#include "format/transformer/writer_assigner.h" +#include "exec/partitioner/writer_assigner.h" namespace doris { #include "common/compile_check_begin.h" @@ -41,8 +41,8 @@ class MergePartitioner final : public PartitionerBase { Status clone(RuntimeState* state, std::unique_ptr& partitioner) override; private: - void _apply_insert_rebalance(const std::vector& ops, - std::vector& insert_hashes, size_t block_bytes) const; + Status _apply_insert_rebalance(const std::vector& ops, + std::vector& insert_hashes, size_t block_bytes) const; void _init_insert_scaling(RuntimeState* state); uint32_t _next_rr_channel() const; Status _clone_expr_ctxs(RuntimeState* state, const VExprContextSPtrs& src, diff --git a/be/src/format/transformer/writer_assigner.h b/be/src/format/transformer/writer_assigner.h deleted file mode 100644 index 173b04867d61b2..00000000000000 --- a/be/src/format/transformer/writer_assigner.h +++ /dev/null @@ -1,127 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "exec/connector/skewed_partition_rebalancer.h" - -namespace doris { -#include "common/compile_check_begin.h" - -class WriterAssigner { -public: - virtual ~WriterAssigner() = default; - - virtual void assign(const std::vector& partition_ids, - const std::vector* mask, size_t rows, size_t block_bytes, - std::vector& writer_ids) = 0; -}; - -class IdentityWriterAssigner final : public WriterAssigner { -public: - void assign(const std::vector& partition_ids, const std::vector* mask, - size_t rows, size_t /*block_bytes*/, std::vector& writer_ids) override { - if (rows == 0) { - return; - } - if (writer_ids.size() != rows && &writer_ids != &partition_ids) { - writer_ids.resize(rows); - } - if (mask == nullptr) { - for (size_t i = 0; i < rows; ++i) { - writer_ids[i] = partition_ids[i]; - } - return; - } - for (size_t i = 0; i < rows; ++i) { - if ((*mask)[i] == 0) { - continue; - } - writer_ids[i] = partition_ids[i]; - } - } -}; - -class SkewedWriterAssigner final : public WriterAssigner { -public: - SkewedWriterAssigner(int partition_count, int task_count, int task_bucket_count, - long min_partition_data_processed_rebalance_threshold, - long min_data_processed_rebalance_threshold) - : _rebalancer(partition_count, task_count, task_bucket_count, - min_partition_data_processed_rebalance_threshold, - min_data_processed_rebalance_threshold), - _partition_row_counts(partition_count, 0), - _partition_writer_ids(partition_count, -1), - _partition_writer_indexes(partition_count, 0) {} - - void assign(const std::vector& partition_ids, const std::vector* mask, - size_t rows, size_t block_bytes, std::vector& writer_ids) override { - if (rows == 0 || _partition_row_counts.empty()) { - return; - } - if (writer_ids.size() != rows && &writer_ids != &partition_ids) { - writer_ids.resize(rows); - } - - std::fill(_partition_row_counts.begin(), _partition_row_counts.end(), 0); - std::fill(_partition_writer_ids.begin(), _partition_writer_ids.end(), -1); - _rebalancer.rebalance(); - - const size_t partition_count = _partition_row_counts.size(); - for (size_t i = 0; i < rows; ++i) { - if (mask != nullptr && (*mask)[i] == 0) { - continue; - } - const uint32_t partition_id = partition_ids[i]; - if (partition_id >= partition_count) { - continue; - } - _partition_row_counts[partition_id] += 1; - int writer_id = _partition_writer_ids[partition_id]; - if (writer_id == -1) { - writer_id = _get_next_writer_id(partition_id); - _partition_writer_ids[partition_id] = writer_id; - } - writer_ids[i] = static_cast(writer_id); - } - - for (size_t i = 0; i < partition_count; ++i) { - if (_partition_row_counts[i] > 0) { - _rebalancer.add_partition_row_count(static_cast(i), _partition_row_counts[i]); - } - } - _rebalancer.add_data_processed(static_cast(block_bytes)); - } - -private: - int _get_next_writer_id(uint32_t partition_id) { - return _rebalancer.get_task_id(partition_id, _partition_writer_indexes[partition_id]++); - } - - SkewedPartitionRebalancer _rebalancer; - std::vector _partition_row_counts; - std::vector _partition_writer_ids; - std::vector _partition_writer_indexes; -}; - -#include "common/compile_check_end.h" -} // namespace doris diff --git a/be/test/exec/partitioner/external_table_sink_hash_partitioner_test.cpp b/be/test/exec/partitioner/external_table_sink_hash_partitioner_test.cpp new file mode 100644 index 00000000000000..ffb7e98307fc24 --- /dev/null +++ b/be/test/exec/partitioner/external_table_sink_hash_partitioner_test.cpp @@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/partitioner/external/external_table_sink_hash_partitioner.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/object_pool.h" +#include "core/block/block.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "runtime/descriptor_helper.h" +#include "runtime/descriptors.h" +#include "testutil/mock/mock_runtime_state.h" + +namespace doris { + +class ExternalTableSinkHashPartitionerTest : public testing::Test { +protected: + void SetUp() override { + TDescriptorTableBuilder table_builder; + TTupleDescriptorBuilder tuple_builder; + tuple_builder.add_slot(TSlotDescriptorBuilder() + .type(TYPE_INT) + .nullable(false) + .column_name("key") + .column_pos(1) + .build()); + tuple_builder.build(&table_builder); + TDescriptorTable thrift_table = table_builder.desc_tbl(); + + DescriptorTbl* descriptor_table = nullptr; + ASSERT_TRUE(DescriptorTbl::create(&_pool, thrift_table, &descriptor_table).ok()); + _state.set_desc_tbl(descriptor_table); + _tuple_id = thrift_table.tupleDescriptors[0].id; + _slot_id = thrift_table.slotDescriptors[0].id; + _row_descriptor = std::make_unique( + *descriptor_table, std::vector {_tuple_id}, std::vector {false}); + } + + TExpr slot_ref() const { + TExprNode node; + node.__set_node_type(TExprNodeType::SLOT_REF); + node.__set_num_children(0); + node.__set_type(create_type_desc(TYPE_INT)); + node.__set_is_nullable(false); + + TSlotRef slot_ref; + slot_ref.__set_slot_id(_slot_id); + slot_ref.__set_tuple_id(_tuple_id); + node.__set_slot_ref(slot_ref); + + TExpr expression; + expression.nodes.emplace_back(std::move(node)); + return expression; + } + + Block block(std::initializer_list values) const { + auto column = ColumnInt32::create(); + for (int32_t value : values) { + column->insert_value(value); + } + Block block; + block.insert( + ColumnWithTypeAndName(std::move(column), std::make_shared(), "key")); + return block; + } + + ObjectPool _pool; + MockRuntimeState _state; + std::unique_ptr _row_descriptor; + TTupleId _tuple_id = -1; + TSlotId _slot_id = -1; +}; + +TEST_F(ExternalTableSinkHashPartitionerTest, DirectHashKeepsOneKeyOnOneWriter) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::DIRECT_HASH); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + ASSERT_TRUE(partitioner.init({slot_ref()}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_descriptor).ok()); + ASSERT_TRUE(partitioner.open(&_state).ok()); + + Block input = block({7, 3, 7, 9, 3}); + ASSERT_TRUE(partitioner.do_partitioning(&_state, &input).ok()); + const auto& channels = partitioner.get_channel_ids(); + ASSERT_EQ(5, channels.size()); + EXPECT_EQ(channels[0], channels[2]); + EXPECT_EQ(channels[1], channels[4]); + EXPECT_EQ(1, input.columns()); + + ASSERT_TRUE(partitioner.close(&_state).ok()); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, OldFePayloadMissingWriterAssignmentFailsClosed) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::DIRECT_HASH); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + + Status status = partitioner.init({slot_ref()}); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("writer assignment is missing"), std::string::npos); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, NewerFeHashAlgorithmFailsClosed) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(static_cast(99)); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + + Status status = partitioner.init({slot_ref()}); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Unsupported external sink hash algorithm 99"), + std::string::npos); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, DirectHashSupportsSkewedWriterAssignment) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::DIRECT_HASH); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::SKEWED); + ExternalTableSinkHashPartitioner partitioner(4, ShuffleHashMethod::CRC32C, info); + ASSERT_TRUE(partitioner.init({slot_ref()}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_descriptor).ok()); + ASSERT_TRUE(partitioner.open(&_state).ok()); + + Block input = block({7, 3, 7, 9, 3}); + ASSERT_TRUE(partitioner.do_partitioning(&_state, &input).ok()); + const auto& channels = partitioner.get_channel_ids(); + ASSERT_EQ(5, channels.size()); + EXPECT_EQ(channels[0], channels[2]); + EXPECT_EQ(channels[1], channels[4]); + for (uint32_t channel : channels) { + EXPECT_LT(channel, 4); + } + + ASSERT_TRUE(partitioner.close(&_state).ok()); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, PaimonFixedBucketUsesSdkCompatibleChannel) { + TPaimonFixedBucketInfo fixed_bucket_info; + fixed_bucket_info.__set_num_buckets(4); + fixed_bucket_info.__set_partition_field_indexes({}); + fixed_bucket_info.__set_bucket_field_indexes({0}); + + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + info.__set_paimon_fixed_bucket_info(fixed_bucket_info); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + ASSERT_TRUE(partitioner.init({slot_ref()}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_descriptor).ok()); + ASSERT_TRUE(partitioner.open(&_state).ok()); + + Block input = block({1, 7, 1}); + ASSERT_TRUE(partitioner.do_partitioning(&_state, &input).ok()); + const auto& channels = partitioner.get_channel_ids(); + ASSERT_EQ(3, channels.size()); + EXPECT_EQ(5, channels[0]); + EXPECT_EQ(channels[0], channels[2]); + + ASSERT_TRUE(partitioner.close(&_state).ok()); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, PaimonFixedBucketIncludesPartitionHash) { + TPaimonFixedBucketInfo fixed_bucket_info; + fixed_bucket_info.__set_num_buckets(4); + fixed_bucket_info.__set_partition_field_indexes({0}); + fixed_bucket_info.__set_bucket_field_indexes({0}); + + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + info.__set_paimon_fixed_bucket_info(fixed_bucket_info); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + ASSERT_TRUE(partitioner.init({slot_ref()}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_descriptor).ok()); + ASSERT_TRUE(partitioner.open(&_state).ok()); + + Block input = block({1, 1}); + ASSERT_TRUE(partitioner.do_partitioning(&_state, &input).ok()); + const auto& channels = partitioner.get_channel_ids(); + ASSERT_EQ(2, channels.size()); + EXPECT_EQ(0, channels[0]); + EXPECT_EQ(channels[0], channels[1]); + + ASSERT_TRUE(partitioner.close(&_state).ok()); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, PaimonFixedBucketRejectsMissingMetadata) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + + Status status = partitioner.init({slot_ref()}); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("routing metadata is missing"), std::string::npos); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, PaimonFixedBucketRejectsEmptyTransformMetadata) { + TPaimonFixedBucketInfo fixed_bucket_info; + fixed_bucket_info.__set_num_buckets(4); + fixed_bucket_info.__set_partition_field_indexes({}); + fixed_bucket_info.__set_bucket_field_indexes({0}); + + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + info.__set_partition_transforms({}); + info.__set_paimon_fixed_bucket_info(fixed_bucket_info); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + + Status status = partitioner.init({slot_ref()}); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains incompatible metadata"), std::string::npos); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, IcebergTransformHashesTransformedValue) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::ICEBERG_TRANSFORM); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::SKEWED); + info.__set_partition_transforms({"truncate[10]"}); + ExternalTableSinkHashPartitioner partitioner(64, ShuffleHashMethod::CRC32, info); + ASSERT_TRUE(partitioner.init({slot_ref()}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_descriptor).ok()); + ASSERT_TRUE(partitioner.open(&_state).ok()); + + Block input = block({11, 19, 20, 29}); + ASSERT_TRUE(partitioner.do_partitioning(&_state, &input).ok()); + const auto& channels = partitioner.get_channel_ids(); + ASSERT_EQ(4, channels.size()); + EXPECT_EQ(channels[0], channels[1]); + EXPECT_EQ(channels[2], channels[3]); + EXPECT_EQ(1, input.columns()); + + ASSERT_TRUE(partitioner.close(&_state).ok()); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, UnsupportedTransformFailsClosed) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::ICEBERG_TRANSFORM); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + info.__set_partition_transforms({"unsupported"}); + ExternalTableSinkHashPartitioner partitioner(4, ShuffleHashMethod::CRC32, info); + ASSERT_TRUE(partitioner.init({slot_ref()}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_descriptor).ok()); + EXPECT_FALSE(partitioner.open(&_state).ok()); +} + +} // namespace doris diff --git a/be/test/exec/partitioner/paimon_native_row_hash_test.cpp b/be/test/exec/partitioner/paimon_native_row_hash_test.cpp new file mode 100644 index 00000000000000..9b60eb0db2843d --- /dev/null +++ b/be/test/exec/partitioner/paimon_native_row_hash_test.cpp @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/partitioner/external/paimon_native_row_hash.h" + +#include + +#include +#include + +namespace doris::paimon_native { + +TEST(PaimonNativeRowHashTest, MatchesPaimonBinaryRowGoldenValues) { + BinaryRowEncoder empty(0); + EXPECT_EQ(empty.bytes().size(), 8); + EXPECT_EQ(empty.hash(), -1670924195); + + BinaryRowEncoder row(1); + ASSERT_TRUE(row.write_int(0, 1)); + EXPECT_EQ(row.bytes().size(), 16); + EXPECT_EQ(row.hash(), 1465514398); + + row.reset(); + ASSERT_TRUE(row.set_null(0)); + EXPECT_EQ(row.hash(), -1748325344); + + row.reset(); + ASSERT_TRUE(row.write_string(0, "abcdefgh")); + EXPECT_EQ(row.bytes().size(), 24); + EXPECT_EQ(row.hash(), -843760178); + + row.reset(); + ASSERT_TRUE(row.write_string(0, "abc")); + EXPECT_EQ(row.bytes().size(), 16); + EXPECT_EQ(row.hash(), -101922419); + + BinaryRowEncoder mixed(2); + ASSERT_TRUE(mixed.write_int(0, 1)); + ASSERT_TRUE(mixed.write_string(1, "abc")); + EXPECT_EQ(mixed.hash(), 261371745); +} + +TEST(PaimonNativeRowHashTest, MatchesDefaultBucketAndChannelComputer) { + ASSERT_EQ(default_bucket(1465514398, 4), 2); + ASSERT_EQ(default_bucket(-7, 4), 3); + ASSERT_FALSE(default_bucket(1, 0).has_value()); + + // Keep these vectors in sync with PaimonNativeRoutingGoldenTest, which computes the same + // ownership using Paimon's DefaultBucketFunction and ChannelComputer directly. + EXPECT_EQ(fixed_bucket_channel(-1670924195, 2, 1), 0); + EXPECT_EQ(fixed_bucket_channel(-1670924195, 2, 2), 1); + EXPECT_EQ(fixed_bucket_channel(-1670924195, 2, 3), 1); + EXPECT_EQ(fixed_bucket_channel(-1670924195, 2, 8), 5); + + EXPECT_EQ(fixed_bucket_channel(1465514398, 1, 1), 0); + EXPECT_EQ(fixed_bucket_channel(1465514398, 1, 2), 1); + EXPECT_EQ(fixed_bucket_channel(1465514398, 1, 3), 2); + EXPECT_EQ(fixed_bucket_channel(1465514398, 1, 4), 3); + EXPECT_EQ(fixed_bucket_channel(1465514398, 1, 8), 7); + + EXPECT_EQ(fixed_bucket_channel(-101922419, 3, 1), 0); + EXPECT_EQ(fixed_bucket_channel(-101922419, 3, 2), 0); + EXPECT_EQ(fixed_bucket_channel(-101922419, 3, 3), 2); + EXPECT_EQ(fixed_bucket_channel(-101922419, 3, 4), 2); + EXPECT_EQ(fixed_bucket_channel(-101922419, 3, 8), 6); + + ASSERT_EQ(fixed_bucket_channel(std::numeric_limits::min(), 1, 8), 0); + ASSERT_FALSE(fixed_bucket_channel(1, 0, 0).has_value()); +} + +} // namespace doris::paimon_native diff --git a/be/test/exec/partitioner/writer_assigner_test.cpp b/be/test/exec/partitioner/writer_assigner_test.cpp new file mode 100644 index 00000000000000..04883d2ed1543f --- /dev/null +++ b/be/test/exec/partitioner/writer_assigner_test.cpp @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/partitioner/writer_assigner.h" + +#include + +#include +#include + +namespace doris { + +TEST(WriterAssignerTest, IdentityPreservesLogicalPartition) { + IdentityWriterAssigner assigner(3); + std::vector partition_ids {2, 0, 1, 2}; + std::vector writer_ids; + + ASSERT_TRUE(assigner.assign(partition_ids, nullptr, partition_ids.size(), 64, writer_ids).ok()); + EXPECT_EQ(partition_ids, writer_ids); +} + +TEST(WriterAssignerTest, IdentityRejectsInvalidLogicalPartition) { + IdentityWriterAssigner assigner(2); + std::vector partition_ids {0, 2}; + std::vector writer_ids; + + Status status = assigner.assign(partition_ids, nullptr, partition_ids.size(), 64, writer_ids); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("exceeds writer count"), std::string::npos); +} + +TEST(WriterAssignerTest, SkewedRejectsInvalidLogicalPartition) { + SkewedWriterAssigner assigner(4, 2, 1, 1, 1); + std::vector partition_ids {0, 4}; + std::vector writer_ids; + + Status status = assigner.assign(partition_ids, nullptr, partition_ids.size(), 64, writer_ids); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("exceeds partition count"), std::string::npos); +} + +} // namespace doris diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonNativeRoutingGoldenTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonNativeRoutingGoldenTest.java new file mode 100644 index 00000000000000..f8a346783b5130 --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonNativeRoutingGoldenTest.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.paimon.bucket.DefaultBucketFunction; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.BinaryRowWriter; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.table.sink.ChannelComputer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.function.Consumer; + +/** + * SDK side of the fixed-bucket native routing golden test. + * + *

The values in this test intentionally match {@code PaimonNativeRowHashTest}. If a Paimon + * upgrade changes BinaryRow, DefaultBucketFunction, or ChannelComputer, the Java test exposes the + * new SDK result while the BE test protects the native implementation from silently diverging. + */ +class PaimonNativeRoutingGoldenTest { + + @Test + void testBinaryRowHashGoldenValues() { + Assertions.assertEquals(-1670924195, BinaryRow.EMPTY_ROW.hashCode()); + Assertions.assertEquals(1465514398, row(1, writer -> writer.writeInt(0, 1)).hashCode()); + Assertions.assertEquals(-1748325344, row(1, writer -> writer.setNullAt(0)).hashCode()); + Assertions.assertEquals(-843760178, row(1, writer -> writer.writeString( + 0, BinaryString.fromString("abcdefgh"))).hashCode()); + Assertions.assertEquals(-101922419, row(1, writer -> writer.writeString( + 0, BinaryString.fromString("abc"))).hashCode()); + Assertions.assertEquals(261371745, row(2, writer -> { + writer.writeInt(0, 1); + writer.writeString(1, BinaryString.fromString("abc")); + }).hashCode()); + } + + @Test + void testBucketAndWriterOwnershipGoldenValues() { + DefaultBucketFunction bucketFunction = new DefaultBucketFunction(); + BinaryRow bucketKey = row(1, writer -> writer.writeInt(0, 1)); + Assertions.assertEquals(2, bucketFunction.bucket(bucketKey, 4)); + + assertChannels(BinaryRow.EMPTY_ROW, 2, new int[][] { + {1, 0}, {2, 1}, {3, 1}, {8, 5} + }); + assertChannels(bucketKey, 1, new int[][] { + {1, 0}, {2, 1}, {3, 2}, {4, 3}, {8, 7} + }); + BinaryRow stringPartition = row(1, writer -> writer.writeString( + 0, BinaryString.fromString("abc"))); + assertChannels(stringPartition, 3, new int[][] { + {1, 0}, {2, 0}, {3, 2}, {4, 2}, {8, 6} + }); + } + + private static BinaryRow row(int arity, Consumer values) { + BinaryRow row = new BinaryRow(arity); + BinaryRowWriter writer = new BinaryRowWriter(row); + values.accept(writer); + writer.complete(); + return row; + } + + private static void assertChannels(BinaryRow partition, int bucket, int[][] expected) { + for (int[] item : expected) { + int writers = item[0]; + int owner = item[1]; + Assertions.assertEquals(owner, ChannelComputer.select(partition, bucket, writers), + "writers=" + writers); + } + } +} diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonWriteSchemaTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonWriteSchemaTest.java index e7136ff6960631..3ed09fbd2094f1 100644 --- a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonWriteSchemaTest.java +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonWriteSchemaTest.java @@ -17,6 +17,7 @@ package org.apache.doris.paimon; +import org.apache.paimon.casting.DefaultValueRow; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalMap; @@ -147,6 +148,26 @@ public void testExplicitNullIsNotReplacedByOmittedFieldDefault() { Assertions.assertTrue(tableRow.isNullAt(1)); } + @Test + public void testPaimonWriterDefaultsExplicitNullRouteFields() { + RowType tableType = new RowType(Arrays.asList( + new DataField(0, "bucket_key", new IntType(), null, "1"), + new DataField(1, "partition_key", new VarCharType(VarCharType.MAX_LENGTH), + null, "default-partition"))); + PaimonWriteSchema schema = PaimonWriteSchema.create( + tableType, new String[] {"bucket_key", "partition_key"}); + + InternalRow tableRow = schema.tableRow(new Object[] {null, null}); + Assertions.assertTrue(tableRow.isNullAt(0)); + Assertions.assertTrue(tableRow.isNullAt(1)); + + DefaultValueRow defaultValueRow = DefaultValueRow.create(tableType); + Assertions.assertNotNull(defaultValueRow); + InternalRow writerRow = defaultValueRow.replaceRow(tableRow); + Assertions.assertEquals(1, writerRow.getInt(0)); + Assertions.assertEquals("default-partition", writerRow.getString(1).toString()); + } + @Test public void testOmittedComplexDefaultsUsePaimonInternalValues() { RowType nestedType = RowType.of(DataTypes.INT(), DataTypes.STRING()); diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 4e535950aae0e3..6432720fb62e7d 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2260,7 +2260,7 @@ public class Config extends ConfigBase { * Max data version of backends serialize block. */ @ConfField(mutable = false) - public static int max_be_exec_version = 11; + public static int max_be_exec_version = 12; /** * Min data version of backends serialize block. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 8be9f51dc1f8cb..5e33caf1cebea1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -86,12 +86,13 @@ import org.apache.doris.nereids.properties.DistributionSpecAllSingleton; import org.apache.doris.nereids.properties.DistributionSpecAny; import org.apache.doris.nereids.properties.DistributionSpecExecutionAny; +import org.apache.doris.nereids.properties.DistributionSpecExternalTableSinkHashPartitioned; +import org.apache.doris.nereids.properties.DistributionSpecExternalTableSinkUnPartitioned; import org.apache.doris.nereids.properties.DistributionSpecGather; import org.apache.doris.nereids.properties.DistributionSpecHash; -import org.apache.doris.nereids.properties.DistributionSpecHiveTableSinkHashPartitioned; -import org.apache.doris.nereids.properties.DistributionSpecHiveTableSinkUnPartitioned; import org.apache.doris.nereids.properties.DistributionSpecMerge; import org.apache.doris.nereids.properties.DistributionSpecOlapTableSinkHashPartitioned; +import org.apache.doris.nereids.properties.DistributionSpecPaimonTableSinkHashPartitioned; import org.apache.doris.nereids.properties.DistributionSpecReplicated; import org.apache.doris.nereids.properties.DistributionSpecStorageAny; import org.apache.doris.nereids.properties.DistributionSpecStorageGather; @@ -245,7 +246,10 @@ import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.StatisticConstants; import org.apache.doris.tablefunction.TableValuedFunctionIf; +import org.apache.doris.thrift.TExternalTableSinkHashAlgorithm; +import org.apache.doris.thrift.TExternalTableSinkWriterAssignment; import org.apache.doris.thrift.TFetchOption; +import org.apache.doris.thrift.TPaimonFixedBucketInfo; import org.apache.doris.thrift.TPartitionType; import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TResultSinkType; @@ -3342,19 +3346,59 @@ private DataPartition toDataPartition(DistributionSpec distributionSpec/* target return new DataPartition(partitionType, partitionExprs); } else if (distributionSpec instanceof DistributionSpecOlapTableSinkHashPartitioned) { return DataPartition.TABLET_ID; - } else if (distributionSpec instanceof DistributionSpecHiveTableSinkHashPartitioned) { - DistributionSpecHiveTableSinkHashPartitioned partitionSpecHash = - (DistributionSpecHiveTableSinkHashPartitioned) distributionSpec; + } else if (distributionSpec instanceof DistributionSpecExternalTableSinkHashPartitioned) { + DistributionSpecExternalTableSinkHashPartitioned externalSpec = + (DistributionSpecExternalTableSinkHashPartitioned) distributionSpec; List partitionExprs = Lists.newArrayList(); - List partitionExprIds = partitionSpecHash.getOutputColExprIds(); - for (ExprId partitionExprId : partitionExprIds) { - if (childOutputIds.contains(partitionExprId)) { - partitionExprs.add(context.findSlotRef(partitionExprId)); + for (ExprId partitionExprId : externalSpec.getOutputColumnExprIds()) { + if (!childOutputIds.contains(partitionExprId)) { + throw new RuntimeException("External table sink partition expression " + + partitionExprId + " is missing from child output"); } + partitionExprs.add(context.findSlotRef(partitionExprId)); } - return new DataPartition(TPartitionType.HIVE_TABLE_SINK_HASH_PARTITIONED, partitionExprs); - } else if (distributionSpec instanceof DistributionSpecHiveTableSinkUnPartitioned) { - return new DataPartition(TPartitionType.HIVE_TABLE_SINK_UNPARTITIONED); + TExternalTableSinkHashAlgorithm algorithm; + switch (externalSpec.getHashAlgorithm()) { + case DIRECT_HASH: + algorithm = TExternalTableSinkHashAlgorithm.DIRECT_HASH; + break; + case ICEBERG_TRANSFORM: + algorithm = TExternalTableSinkHashAlgorithm.ICEBERG_TRANSFORM; + break; + case PAIMON_FIXED_BUCKET: + algorithm = TExternalTableSinkHashAlgorithm.PAIMON_FIXED_BUCKET; + break; + default: + throw new RuntimeException("Unsupported external table sink hash algorithm: " + + externalSpec.getHashAlgorithm()); + } + TExternalTableSinkWriterAssignment writerAssignment; + switch (externalSpec.getWriterAssignment()) { + case IDENTITY: + writerAssignment = TExternalTableSinkWriterAssignment.IDENTITY; + break; + case SKEWED: + writerAssignment = TExternalTableSinkWriterAssignment.SKEWED; + break; + default: + throw new RuntimeException("Unsupported external table sink writer assignment: " + + externalSpec.getWriterAssignment()); + } + TPaimonFixedBucketInfo paimonFixedBucketInfo = null; + if (externalSpec instanceof DistributionSpecPaimonTableSinkHashPartitioned) { + DistributionSpecPaimonTableSinkHashPartitioned paimonSpec + = (DistributionSpecPaimonTableSinkHashPartitioned) externalSpec; + paimonFixedBucketInfo = new TPaimonFixedBucketInfo(); + paimonFixedBucketInfo.setNumBuckets(paimonSpec.getNumBuckets()); + paimonFixedBucketInfo.setPartitionFieldIndexes( + paimonSpec.getPartitionFieldIndexes()); + paimonFixedBucketInfo.setBucketFieldIndexes(paimonSpec.getBucketFieldIndexes()); + } + return new DataPartition(TPartitionType.EXTERNAL_TABLE_SINK_HASH_PARTITIONED, + partitionExprs, algorithm, writerAssignment, + externalSpec.getPartitionTransforms(), paimonFixedBucketInfo); + } else if (distributionSpec instanceof DistributionSpecExternalTableSinkUnPartitioned) { + return new DataPartition(TPartitionType.EXTERNAL_TABLE_SINK_UNPARTITIONED); } else if (distributionSpec instanceof DistributionSpecMerge) { DistributionSpecMerge mergeSpec = (DistributionSpecMerge) distributionSpec; Expr operationExpr = context.findSlotRef(mergeSpec.getOperationExprId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecExternalTableSinkHashPartitioned.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecExternalTableSinkHashPartitioned.java new file mode 100644 index 00000000000000..bdf3caa55140fe --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecExternalTableSinkHashPartitioned.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +import org.apache.doris.nereids.trees.expressions.ExprId; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +/** Common logical-partition and writer-assignment contract for external table sink writers. */ +public abstract class DistributionSpecExternalTableSinkHashPartitioned extends DistributionSpec { + public static final int MIN_BE_EXEC_VERSION = 12; + + private final ImmutableList outputColumnExprIds; + + protected DistributionSpecExternalTableSinkHashPartitioned(List outputColumnExprIds) { + this.outputColumnExprIds = ImmutableList.copyOf( + Objects.requireNonNull(outputColumnExprIds, "outputColumnExprIds should not be null")); + if (this.outputColumnExprIds.isEmpty()) { + throw new IllegalArgumentException("outputColumnExprIds should not be empty"); + } + } + + public List getOutputColumnExprIds() { + return outputColumnExprIds; + } + + public abstract HashAlgorithm getHashAlgorithm(); + + public abstract WriterAssignment getWriterAssignment(); + + public List getPartitionTransforms() { + return ImmutableList.of(); + } + + @Override + public boolean satisfy(DistributionSpec required) { + return required instanceof DistributionSpecAny || equals(required); + } + + @Override + public String shapeInfo() { + return getClass().getSimpleName(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + DistributionSpecExternalTableSinkHashPartitioned that = + (DistributionSpecExternalTableSinkHashPartitioned) other; + return outputColumnExprIds.equals(that.outputColumnExprIds) + && getHashAlgorithm() == that.getHashAlgorithm() + && getWriterAssignment() == that.getWriterAssignment() + && getPartitionTransforms().equals(that.getPartitionTransforms()); + } + + @Override + public int hashCode() { + return Objects.hash(getClass(), outputColumnExprIds, getHashAlgorithm(), getWriterAssignment(), + getPartitionTransforms()); + } + + /** Algorithms shared by FE planning and the external sink exchange protocol. */ + public enum HashAlgorithm { + DIRECT_HASH, + ICEBERG_TRANSFORM, + PAIMON_FIXED_BUCKET + } + + /** Maps logical sink partitions to Doris exchange writers. */ + public enum WriterAssignment { + IDENTITY, + SKEWED + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecExternalTableSinkUnPartitioned.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecExternalTableSinkUnPartitioned.java new file mode 100644 index 00000000000000..00d4901de885c9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecExternalTableSinkUnPartitioned.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +/** + * Adaptive writer distribution for external sinks without an ownership key. + * + *

This preserves the original Hive non-partitioned ScaleWriter behavior. Hive and Iceberg + * choose it explicitly in the first phase of the common sink-exchange work; other external + * formats continue to share it until their own distribution contracts are modeled. + */ +public final class DistributionSpecExternalTableSinkUnPartitioned extends DistributionSpec { + + public static final DistributionSpecExternalTableSinkUnPartitioned INSTANCE = + new DistributionSpecExternalTableSinkUnPartitioned(); + + private DistributionSpecExternalTableSinkUnPartitioned() { + } + + @Override + public boolean satisfy(DistributionSpec required) { + return required instanceof DistributionSpecExternalTableSinkUnPartitioned; + } + + @Override + public String shapeInfo() { + return "DistributionSpecExternalTableSinkUnPartitioned"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHiveTableSinkHashPartitioned.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHiveTableSinkHashPartitioned.java index 4275bb53dd85ca..c6f0da021e0d51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHiveTableSinkHashPartitioned.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHiveTableSinkHashPartitioned.java @@ -21,27 +21,22 @@ import java.util.List; -/** - * use for shuffle data by partition keys before sink. - */ -public class DistributionSpecHiveTableSinkHashPartitioned extends DistributionSpec { +/** Hash Hive partition values and scale hot partitions across writers. */ +public final class DistributionSpecHiveTableSinkHashPartitioned + extends DistributionSpecExternalTableSinkHashPartitioned { - private List outputColExprIds; - - public DistributionSpecHiveTableSinkHashPartitioned() { - super(); - } - - public List getOutputColExprIds() { - return outputColExprIds; + /** Create a Hive ownership distribution from partition columns. */ + public DistributionSpecHiveTableSinkHashPartitioned(List outputColumnExprIds) { + super(outputColumnExprIds); } - public void setOutputColExprIds(List outputColExprIds) { - this.outputColExprIds = outputColExprIds; + @Override + public HashAlgorithm getHashAlgorithm() { + return HashAlgorithm.DIRECT_HASH; } @Override - public boolean satisfy(DistributionSpec other) { - return other instanceof DistributionSpecHiveTableSinkHashPartitioned; + public WriterAssignment getWriterAssignment() { + return WriterAssignment.SKEWED; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHiveTableSinkUnPartitioned.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHiveTableSinkUnPartitioned.java deleted file mode 100644 index 1bca3092424d69..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHiveTableSinkUnPartitioned.java +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.properties; - -/** - * use for Round Robin by data sink. - */ -public class DistributionSpecHiveTableSinkUnPartitioned extends DistributionSpec { - - public static final DistributionSpecHiveTableSinkUnPartitioned INSTANCE = - new DistributionSpecHiveTableSinkUnPartitioned(); - - private DistributionSpecHiveTableSinkUnPartitioned() { - super(); - } - - @Override - public boolean satisfy(DistributionSpec other) { - return other instanceof DistributionSpecHiveTableSinkUnPartitioned; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecIcebergTableSinkHashPartitioned.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecIcebergTableSinkHashPartitioned.java new file mode 100644 index 00000000000000..2c6071678f39d4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecIcebergTableSinkHashPartitioned.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +import org.apache.doris.nereids.trees.expressions.ExprId; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +/** Hash final Iceberg partition values to a unique external table sink writer. */ +public final class DistributionSpecIcebergTableSinkHashPartitioned + extends DistributionSpecExternalTableSinkHashPartitioned { + + private final ImmutableList partitionTransforms; + + /** Create an Iceberg ownership distribution from source columns and positional transforms. */ + public DistributionSpecIcebergTableSinkHashPartitioned( + List outputColumnExprIds, List partitionTransforms) { + super(outputColumnExprIds); + this.partitionTransforms = ImmutableList.copyOf( + Objects.requireNonNull(partitionTransforms, "partitionTransforms should not be null")); + if (this.partitionTransforms.size() != outputColumnExprIds.size()) { + throw new IllegalArgumentException( + "partitionTransforms must match outputColumnExprIds"); + } + } + + @Override + public HashAlgorithm getHashAlgorithm() { + return HashAlgorithm.ICEBERG_TRANSFORM; + } + + @Override + public WriterAssignment getWriterAssignment() { + // Iceberg ordinary INSERT historically reused Hive ScaleWriter. Keep that concurrency + // behavior while making the logical partition calculation match Iceberg transforms. + return WriterAssignment.SKEWED; + } + + @Override + public List getPartitionTransforms() { + return partitionTransforms; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecMaxComputeTableSinkHashPartitioned.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecMaxComputeTableSinkHashPartitioned.java new file mode 100644 index 00000000000000..3c4607e810581c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecMaxComputeTableSinkHashPartitioned.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +import org.apache.doris.nereids.trees.expressions.ExprId; + +import java.util.List; + +/** Hash MaxCompute partition values using the legacy Hive ScaleWriter behavior. */ +public final class DistributionSpecMaxComputeTableSinkHashPartitioned + extends DistributionSpecExternalTableSinkHashPartitioned { + + /** Create a MaxCompute ownership distribution from dynamic partition columns. */ + public DistributionSpecMaxComputeTableSinkHashPartitioned(List outputColumnExprIds) { + super(outputColumnExprIds); + } + + @Override + public HashAlgorithm getHashAlgorithm() { + return HashAlgorithm.DIRECT_HASH; + } + + @Override + public WriterAssignment getWriterAssignment() { + // Keep the pre-refactoring behavior for external formats that have not yet defined + // their own writer-ownership contract. Only Hive and Iceberg are modeled explicitly + // in this phase. + return WriterAssignment.SKEWED; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecPaimonTableSinkHashPartitioned.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecPaimonTableSinkHashPartitioned.java new file mode 100644 index 00000000000000..da494529cc56ba --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecPaimonTableSinkHashPartitioned.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +import org.apache.doris.nereids.trees.expressions.ExprId; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +/** Paimon-compatible partition and fixed-bucket routing with stable writer ownership. */ +public final class DistributionSpecPaimonTableSinkHashPartitioned + extends DistributionSpecExternalTableSinkHashPartitioned { + + private final int numBuckets; + private final ImmutableList partitionFieldIndexes; + private final ImmutableList bucketFieldIndexes; + + /** Create a Paimon HASH_FIXED distribution from an ordered list of route expressions. */ + public DistributionSpecPaimonTableSinkHashPartitioned(List routeExprIds, + int numBuckets, List partitionFieldIndexes, List bucketFieldIndexes) { + super(routeExprIds); + if (numBuckets <= 0) { + throw new IllegalArgumentException("Paimon bucket count must be positive"); + } + this.numBuckets = numBuckets; + this.partitionFieldIndexes = checkedIndexes(partitionFieldIndexes, routeExprIds.size(), false); + this.bucketFieldIndexes = checkedIndexes(bucketFieldIndexes, routeExprIds.size(), true); + } + + private static ImmutableList checkedIndexes( + List indexes, int exprCount, boolean requireNonEmpty) { + Objects.requireNonNull(indexes, "Paimon field indexes must not be null"); + if (requireNonEmpty && indexes.isEmpty()) { + throw new IllegalArgumentException("Paimon bucket field indexes must not be empty"); + } + for (Integer index : indexes) { + if (index == null || index < 0 || index >= exprCount) { + throw new IllegalArgumentException("Invalid Paimon route expression index: " + index); + } + } + return ImmutableList.copyOf(indexes); + } + + public int getNumBuckets() { + return numBuckets; + } + + public List getPartitionFieldIndexes() { + return partitionFieldIndexes; + } + + public List getBucketFieldIndexes() { + return bucketFieldIndexes; + } + + @Override + public HashAlgorithm getHashAlgorithm() { + return HashAlgorithm.PAIMON_FIXED_BUCKET; + } + + @Override + public WriterAssignment getWriterAssignment() { + return WriterAssignment.IDENTITY; + } + + @Override + public boolean equals(Object other) { + if (!super.equals(other)) { + return false; + } + DistributionSpecPaimonTableSinkHashPartitioned that + = (DistributionSpecPaimonTableSinkHashPartitioned) other; + return numBuckets == that.numBuckets + && partitionFieldIndexes.equals(that.partitionFieldIndexes) + && bucketFieldIndexes.equals(that.bucketFieldIndexes); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), numBuckets, partitionFieldIndexes, bucketFieldIndexes); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/PhysicalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/PhysicalProperties.java index c28d6ac3cb4d47..36a1ba0029c3f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/PhysicalProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/PhysicalProperties.java @@ -49,8 +49,8 @@ public class PhysicalProperties { public static PhysicalProperties TABLET_ID_SHUFFLE = new PhysicalProperties(DistributionSpecOlapTableSinkHashPartitioned.INSTANCE); - public static PhysicalProperties SINK_RANDOM_PARTITIONED - = new PhysicalProperties(DistributionSpecHiveTableSinkUnPartitioned.INSTANCE); + public static PhysicalProperties EXTERNAL_TABLE_SINK_UNPARTITIONED + = new PhysicalProperties(DistributionSpecExternalTableSinkUnPartitioned.INSTANCE); // gather then broadcast to all BE with exact one instance public static PhysicalProperties ALL_SINGLETON = new PhysicalProperties(DistributionSpecAllSingleton.INSTANCE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHiveTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHiveTableSink.java index 5d8edcab45c21d..178656b6bfd3a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHiveTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHiveTableSink.java @@ -123,11 +123,8 @@ public PhysicalProperties getRequirePhysicalProperties() { List exprIds = columnIdx.stream() .map(idx -> child().getOutput().get(idx).getExprId()) .collect(Collectors.toList()); - DistributionSpecHiveTableSinkHashPartitioned shuffleInfo - = new DistributionSpecHiveTableSinkHashPartitioned(); - shuffleInfo.setOutputColExprIds(exprIds); - return new PhysicalProperties(shuffleInfo); + return new PhysicalProperties(new DistributionSpecHiveTableSinkHashPartitioned(exprIds)); } - return PhysicalProperties.SINK_RANDOM_PARTITIONED; + return PhysicalProperties.EXTERNAL_TABLE_SINK_UNPARTITIONED; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergTableSink.java index cfcd4ac0a73299..0b749999f3b467 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergTableSink.java @@ -18,14 +18,18 @@ package org.apache.doris.nereids.trees.plans.physical; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.DistributionSpecHiveTableSinkHashPartitioned; +import org.apache.doris.nereids.properties.DistributionSpecExternalTableSinkHashPartitioned; +import org.apache.doris.nereids.properties.DistributionSpecIcebergTableSinkHashPartitioned; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; @@ -38,10 +42,10 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; +import java.util.TreeMap; /** physical iceberg sink */ public class PhysicalIcebergTableSink extends PhysicalBaseExternalTableSink { @@ -135,31 +139,99 @@ public PhysicalProperties getRequirePhysicalProperties() { return PhysicalProperties.GATHER; } - Set partitionNames = new java.util.TreeSet<>(String.CASE_INSENSITIVE_ORDER); + if (targetIcebergTable.spec().isPartitioned()) { + if (Config.be_exec_version + < DistributionSpecExternalTableSinkHashPartitioned.MIN_BE_EXEC_VERSION) { + return PhysicalProperties.GATHER; + } + DistributionSpecIcebergTableSinkHashPartitioned distributionSpec + = buildPartitionDistributionSpec(); + // An unsupported transform must not silently hash its raw source column as though it + // were the final Iceberg partition value. GATHER is the safe fallback until the + // transform is supported. + return distributionSpec == null + ? PhysicalProperties.GATHER + : new PhysicalProperties(distributionSpec); + } + return PhysicalProperties.EXTERNAL_TABLE_SINK_UNPARTITIONED; + } + + private DistributionSpecIcebergTableSinkHashPartitioned buildPartitionDistributionSpec() { + Map outputSlotsByName = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (Slot outputSlot : child().getOutput()) { + if (outputSlotsByName.put(outputSlot.getName(), outputSlot) != null) { + return null; + } + } + + List sourceExprIds = new ArrayList<>(); + List transforms = new ArrayList<>(); for (PartitionField field : targetIcebergTable.spec().fields()) { Types.NestedField sourceField = targetIcebergTable.schema().findField(field.sourceId()); - if (sourceField != null) { - partitionNames.add(sourceField.name()); + if (sourceField == null) { + return null; + } + Slot sourceSlot = outputSlotsByName.get(sourceField.name()); + String transform = field.transform().toString(); + if (sourceSlot == null || !supportsPartitionTransform( + transform, sourceSlot.getDataType().toCatalogDataType().getPrimitiveType())) { + return null; + } + sourceExprIds.add(sourceSlot.getExprId()); + transforms.add(transform); + } + if (sourceExprIds.isEmpty()) { + return null; + } + return new DistributionSpecIcebergTableSinkHashPartitioned(sourceExprIds, transforms); + } + + private boolean supportsPartitionTransform(String transform, PrimitiveType sourceType) { + if ("identity".equals(transform) || "void".equals(transform)) { + return true; + } + if ("year".equals(transform) || "month".equals(transform) || "day".equals(transform)) { + return sourceType == PrimitiveType.DATEV2 || sourceType == PrimitiveType.DATETIMEV2; + } + if ("hour".equals(transform)) { + return sourceType == PrimitiveType.DATETIMEV2; + } + if (transform.startsWith("bucket[") && transform.endsWith("]")) { + switch (sourceType) { + case INT: + case BIGINT: + case VARCHAR: + case CHAR: + case STRING: + case DATEV2: + case DATETIMEV2: + case DECIMALV2: + case DECIMAL32: + case DECIMAL64: + case DECIMAL128: + case DECIMAL256: + return true; + default: + return false; } } - if (!partitionNames.isEmpty()) { - List columnIdx = new ArrayList<>(); - // Sink columns are bound from targetIcebergTable; using refreshable table metadata - // here could shuffle rows with a different partition spec than the writer serializes. - for (int i = 0; i < child().getOutput().size(); i++) { - if (partitionNames.contains(child().getOutput().get(i).getName())) { - columnIdx.add(i); - } + if (transform.startsWith("truncate[") && transform.endsWith("]")) { + switch (sourceType) { + case INT: + case BIGINT: + case VARCHAR: + case CHAR: + case STRING: + case DECIMALV2: + case DECIMAL32: + case DECIMAL64: + case DECIMAL128: + case DECIMAL256: + return true; + default: + return false; } - // mapping partition id - List exprIds = columnIdx.stream() - .map(idx -> child().getOutput().get(idx).getExprId()) - .collect(Collectors.toList()); - DistributionSpecHiveTableSinkHashPartitioned shuffleInfo - = new DistributionSpecHiveTableSinkHashPartitioned(); - shuffleInfo.setOutputColExprIds(exprIds); - return new PhysicalProperties(shuffleInfo); } - return PhysicalProperties.SINK_RANDOM_PARTITIONED; + return false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalMaxComputeTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalMaxComputeTableSink.java index bc0f6fc7f31f14..1f03e3e131b22f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalMaxComputeTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalMaxComputeTableSink.java @@ -21,7 +21,7 @@ import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase; import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.DistributionSpecHiveTableSinkHashPartitioned; +import org.apache.doris.nereids.properties.DistributionSpecMaxComputeTableSinkHashPartitioned; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.MustLocalSortOrderSpec; import org.apache.doris.nereids.properties.OrderKey; @@ -123,7 +123,9 @@ public PhysicalProperties getRequirePhysicalProperties() { boolean hasDynamicPartition = partitionNames.stream().anyMatch(colNames::contains); if (!hasDynamicPartition) { // All partition columns are statically specified, no sort needed - return PhysicalProperties.SINK_RANDOM_PARTITIONED; + // Keep sharing the original Hive-style non-partitioned writer scaling until + // MaxCompute defines an independent sink-exchange contract. + return PhysicalProperties.EXTERNAL_TABLE_SINK_UNPARTITIONED; } List columnIdx = new ArrayList<>(); @@ -137,9 +139,8 @@ public PhysicalProperties getRequirePhysicalProperties() { List exprIds = columnIdx.stream() .map(idx -> child().getOutput().get(idx).getExprId()) .collect(Collectors.toList()); - DistributionSpecHiveTableSinkHashPartitioned shuffleInfo - = new DistributionSpecHiveTableSinkHashPartitioned(); - shuffleInfo.setOutputColExprIds(exprIds); + DistributionSpecMaxComputeTableSinkHashPartitioned shuffleInfo + = new DistributionSpecMaxComputeTableSinkHashPartitioned(exprIds); // Require local sort by partition columns so that rows for the same partition // are grouped together. MaxCompute Storage API streams dynamic partition data // and will close a partition writer once it sees a different partition; @@ -150,6 +151,8 @@ public PhysicalProperties getRequirePhysicalProperties() { return new PhysicalProperties(shuffleInfo) .withOrderSpec(new MustLocalSortOrderSpec(orderKeys)); } - return PhysicalProperties.SINK_RANDOM_PARTITIONED; + // Keep sharing the original Hive-style non-partitioned writer scaling until + // MaxCompute defines an independent sink-exchange contract. + return PhysicalProperties.EXTERNAL_TABLE_SINK_UNPARTITIONED; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSink.java index 824625de3cb163..0e4ad54143a36b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSink.java @@ -18,10 +18,13 @@ package org.apache.doris.nereids.trees.plans.physical; import org.apache.doris.catalog.Column; +import org.apache.doris.common.Config; import org.apache.doris.datasource.paimon.PaimonExternalDatabase; import org.apache.doris.datasource.paimon.PaimonWriteTarget; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.DistributionSpecExternalTableSinkHashPartitioned; import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; +import org.apache.doris.nereids.properties.DistributionSpecPaimonTableSinkHashPartitioned; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -30,12 +33,17 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.statistics.Statistics; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import org.apache.paimon.CoreOptions; +import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypeRoot; import java.util.ArrayList; import java.util.List; @@ -58,7 +66,7 @@ public PhysicalPaimonTableSink(PaimonExternalDatabase database, LogicalProperties logicalProperties, CHILD_TYPE child) { this(database, writeTarget, cols, outputExprs, groupExpression, logicalProperties, - PhysicalProperties.SINK_RANDOM_PARTITIONED, null, child); + PhysicalProperties.EXTERNAL_TABLE_SINK_UNPARTITIONED, null, child); } public PhysicalPaimonTableSink(PaimonExternalDatabase database, @@ -108,13 +116,26 @@ public PhysicalPaimonTableSink withPhysicalPropertiesAndStats( @Override public PhysicalProperties getRequirePhysicalProperties() { FileStoreTable paimonTable = writeTarget.getTable(); + DistributionSpecPaimonTableSinkHashPartitioned fixedBucketSpec + = buildFixedBucketDistributionSpec(paimonTable); + if (fixedBucketSpec != null) { + return new PhysicalProperties(fixedBucketSpec); + } + if (paimonTable.bucketMode() == BucketMode.BUCKET_UNAWARE) { + // Bucket-unaware tables are append-only and Paimon deliberately gives each writer + // an empty, no-compaction file store. No bucket ownership is shared, so explicit + // random distribution is safe. ScaleWriter can be evaluated independently later. + return PhysicalProperties.EXECUTION_ANY; + } if (requiresSingleWriter(paimonTable)) { return PhysicalProperties.GATHER; } List primaryKeys = paimonTable.primaryKeys(); if (primaryKeys.isEmpty()) { - return PhysicalProperties.SINK_RANDOM_PARTITIONED; + // Retain adaptive distribution for other append modes which are already safe without + // bucket ownership. BUCKET_UNAWARE is handled above with explicit random routing. + return PhysicalProperties.EXTERNAL_TABLE_SINK_UNPARTITIONED; } List outputSlots = child().getOutput(); @@ -137,26 +158,29 @@ public PhysicalProperties getRequirePhysicalProperties() { /** * Whether this sink must use one writer to preserve Paimon write semantics. * - *

For dynamic bucket tables, GATHER guarantees one writer only within the current - * INSERT. Paimon does not support concurrent write jobs to the same partition, and Doris - * does not add a process-local lease which could not coordinate FE failover or external - * Flink/Spark writers. Concurrent INSERTs into a dynamic bucket table are therefore - * unsupported. + *

HASH_DYNAMIC and KEY_DYNAMIC gather to one writer within the current INSERT because + * their stateful assigners cannot safely be shared. This does not serialize independent jobs; + * concurrent jobs writing the same dynamic-bucket partition remain unsupported. Fixed-bucket + * tables use concurrent writers only when Doris can reproduce Paimon's stateless route. */ public boolean requiresSingleWriter() { + if (buildFixedBucketDistributionSpec(writeTarget.getTable()) != null) { + return false; + } return requiresSingleWriter(writeTarget.getTable()); } static boolean requiresSingleWriter(FileStoreTable paimonTable) { BucketMode bucketMode = paimonTable.bucketMode(); CoreOptions coreOptions = CoreOptions.fromMap(paimonTable.options()); + if (bucketMode == BucketMode.BUCKET_UNAWARE) { + return false; + } if (bucketMode == BucketMode.HASH_DYNAMIC || bucketMode == BucketMode.KEY_DYNAMIC - // Until Doris has a Paimon bucket-aware exchange, a fixed-bucket - // primary-key table must not let independent writers own the same bucket. - // An append-only writer has the same ownership requirement while automatic - // compaction is enabled, because two writers can restore and replace the same - // existing files in one Doris transaction. + // When the native fixed-bucket route is unavailable, a primary-key table must + // not let independent writers own the same bucket. An append-only writer has the + // same fallback requirement while automatic compaction is enabled. || (bucketMode == BucketMode.HASH_FIXED && (!paimonTable.primaryKeys().isEmpty() || !coreOptions.writeOnly()))) { return true; @@ -168,6 +192,109 @@ static boolean requiresSingleWriter(FileStoreTable paimonTable) { == CoreOptions.ChangelogProducer.FULL_COMPACTION); } + private DistributionSpecPaimonTableSinkHashPartitioned buildFixedBucketDistributionSpec( + FileStoreTable paimonTable) { + if (Config.be_exec_version + < DistributionSpecExternalTableSinkHashPartitioned.MIN_BE_EXEC_VERSION) { + return null; + } + return buildFixedBucketDistributionSpec(paimonTable, cols, child().getOutput()); + } + + static DistributionSpecPaimonTableSinkHashPartitioned buildFixedBucketDistributionSpec( + FileStoreTable paimonTable, List sinkColumns, List sinkOutput) { + if (paimonTable.bucketMode() != BucketMode.HASH_FIXED) { + return null; + } + + TableSchema schema = paimonTable.schema(); + CoreOptions coreOptions = CoreOptions.fromMap(schema.options()); + if (coreOptions.bucketFunctionType() != CoreOptions.BucketFunctionType.DEFAULT) { + return null; + } + + if (schema.numBuckets() <= 0 || schema.bucketKeys().isEmpty() + || sinkColumns.size() != sinkOutput.size()) { + return null; + } + + Map outputByName = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (int i = 0; i < sinkColumns.size(); i++) { + if (outputByName.put(sinkColumns.get(i).getName(), sinkOutput.get(i)) != null) { + return null; + } + } + Map fieldsByName = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (DataField field : schema.fields()) { + fieldsByName.put(field.name(), field); + } + + List routeExprIds = new ArrayList<>(); + Map routeIndexes = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + List partitionFieldIndexes = appendRouteFields( + schema.partitionKeys(), outputByName, fieldsByName, routeExprIds, routeIndexes); + List bucketFieldIndexes = appendRouteFields( + schema.bucketKeys(), outputByName, fieldsByName, routeExprIds, routeIndexes); + if (partitionFieldIndexes == null || bucketFieldIndexes == null + || bucketFieldIndexes.isEmpty()) { + return null; + } + return new DistributionSpecPaimonTableSinkHashPartitioned( + routeExprIds, schema.numBuckets(), partitionFieldIndexes, bucketFieldIndexes); + } + + private static List appendRouteFields(List fieldNames, + Map outputByName, Map fieldsByName, + List routeExprIds, Map routeIndexes) { + ImmutableList.Builder indexes = ImmutableList.builder(); + for (String fieldName : fieldNames) { + Slot slot = outputByName.get(fieldName); + DataField field = fieldsByName.get(fieldName); + if (slot == null || field == null + // TableWriteImpl applies schema defaults before extracting the final route, + // while the native Exchange sees the original value, including explicit NULL. + || field.defaultValue() != null + || !supportsNativeRouting(field.type().getTypeRoot(), slot.getDataType())) { + return null; + } + Integer index = routeIndexes.get(fieldName); + if (index == null) { + index = routeExprIds.size(); + routeExprIds.add(slot.getExprId()); + routeIndexes.put(fieldName, index); + } + indexes.add(index); + } + return indexes.build(); + } + + private static boolean supportsNativeRouting(DataTypeRoot paimonType, DataType dorisType) { + switch (paimonType) { + case BOOLEAN: + return dorisType.isBooleanType(); + case TINYINT: + return dorisType.isTinyIntType(); + case SMALLINT: + return dorisType.isSmallIntType(); + case INTEGER: + return dorisType.isIntegerType(); + case BIGINT: + return dorisType.isBigIntType(); + case FLOAT: + return dorisType.isFloatType(); + case DOUBLE: + return dorisType.isDoubleType(); + case CHAR: + case VARCHAR: + return dorisType.isStringLikeType(); + case BINARY: + case VARBINARY: + return dorisType.isStringLikeType() || dorisType.isVarBinaryType(); + default: + return false; + } + } + public PaimonWriteTarget getWriteTarget() { return writeTarget; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java index a41b901f6e5877..8e652821c7663b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java @@ -23,8 +23,12 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.thrift.TDataPartition; import org.apache.doris.thrift.TExplainLevel; +import org.apache.doris.thrift.TExternalTableSinkHashAlgorithm; +import org.apache.doris.thrift.TExternalTableSinkHashPartitionInfo; +import org.apache.doris.thrift.TExternalTableSinkWriterAssignment; import org.apache.doris.thrift.TIcebergPartitionField; import org.apache.doris.thrift.TMergePartitionInfo; +import org.apache.doris.thrift.TPaimonFixedBucketInfo; import org.apache.doris.thrift.TPartitionType; import com.google.common.base.Joiner; @@ -52,22 +56,50 @@ public class DataPartition { // for hash partition: exprs used to compute hash value private ImmutableList partitionExprs; private MergePartitionInfo mergePartitionInfo; + private TExternalTableSinkHashAlgorithm externalHashAlgorithm; + private TExternalTableSinkWriterAssignment externalWriterAssignment; + private ImmutableList externalPartitionTransforms = ImmutableList.of(); + private TPaimonFixedBucketInfo paimonFixedBucketInfo; public DataPartition(TPartitionType type, List exprs) { Preconditions.checkNotNull(exprs); Preconditions.checkState(!exprs.isEmpty()); Preconditions.checkState(type == TPartitionType.HASH_PARTITIONED || type == TPartitionType.RANGE_PARTITIONED - || type == TPartitionType.HIVE_TABLE_SINK_HASH_PARTITIONED + || type == TPartitionType.EXTERNAL_TABLE_SINK_HASH_PARTITIONED || type == TPartitionType.BUCKET_SHFFULE_HASH_PARTITIONED); this.type = type; this.partitionExprs = ImmutableList.copyOf(exprs); } + public DataPartition(TPartitionType type, List exprs, + TExternalTableSinkHashAlgorithm hashAlgorithm, + TExternalTableSinkWriterAssignment writerAssignment, + List partitionTransforms, + TPaimonFixedBucketInfo paimonFixedBucketInfo) { + this(type, exprs); + Preconditions.checkState(type == TPartitionType.EXTERNAL_TABLE_SINK_HASH_PARTITIONED); + this.externalHashAlgorithm = Preconditions.checkNotNull(hashAlgorithm); + this.externalWriterAssignment = Preconditions.checkNotNull(writerAssignment); + Preconditions.checkNotNull(partitionTransforms); + if (hashAlgorithm == TExternalTableSinkHashAlgorithm.ICEBERG_TRANSFORM) { + Preconditions.checkState(partitionTransforms.size() == exprs.size()); + Preconditions.checkState(paimonFixedBucketInfo == null); + } else if (hashAlgorithm == TExternalTableSinkHashAlgorithm.PAIMON_FIXED_BUCKET) { + Preconditions.checkState(partitionTransforms.isEmpty()); + Preconditions.checkNotNull(paimonFixedBucketInfo); + } else { + Preconditions.checkState(partitionTransforms.isEmpty()); + Preconditions.checkState(paimonFixedBucketInfo == null); + } + this.externalPartitionTransforms = ImmutableList.copyOf(partitionTransforms); + this.paimonFixedBucketInfo = paimonFixedBucketInfo; + } + public DataPartition(TPartitionType type) { Preconditions.checkState(type == TPartitionType.UNPARTITIONED || type == TPartitionType.RANDOM - || type == TPartitionType.HIVE_TABLE_SINK_UNPARTITIONED + || type == TPartitionType.EXTERNAL_TABLE_SINK_UNPARTITIONED || type == TPartitionType.OLAP_TABLE_SINK_HASH_PARTITIONED); this.type = type; this.partitionExprs = ImmutableList.of(); @@ -107,6 +139,18 @@ public TDataPartition toThrift() { if (mergePartitionInfo != null) { result.setMergePartitionInfo(mergePartitionInfo.toThrift()); } + if (externalHashAlgorithm != null) { + TExternalTableSinkHashPartitionInfo info = new TExternalTableSinkHashPartitionInfo(); + info.setAlgorithm(externalHashAlgorithm); + info.setWriterAssignment(externalWriterAssignment); + if (!externalPartitionTransforms.isEmpty()) { + info.setPartitionTransforms(externalPartitionTransforms); + } + if (paimonFixedBucketInfo != null) { + info.setPaimonFixedBucketInfo(paimonFixedBucketInfo); + } + result.setExternalTableSinkHashPartitionInfo(info); + } return result; } @@ -143,8 +187,11 @@ public String getExplainString(TExplainLevel explainLevel) { } } else if (!partitionExprs.isEmpty()) { List strings = Lists.newArrayList(); - for (Expr expr : partitionExprs) { - strings.add(expr.toSql()); + for (int i = 0; i < partitionExprs.size(); i++) { + Expr expr = partitionExprs.get(i); + strings.add(externalPartitionTransforms.isEmpty() + ? expr.toSql() + : externalPartitionTransforms.get(i) + "(" + expr.toSql() + ")"); } str.append(": ").append(Joiner.on(", ").join(strings)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PaimonTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PaimonTableSink.java index a6f2e095c3ba44..ff8e632c024406 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PaimonTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PaimonTableSink.java @@ -50,7 +50,8 @@ * metadata, Hadoop authentication config, transaction identity, write mode, * and sink column names. * - * v1: single-writer architecture; partition/bucket routing delegated to SDK. + * The upstream Exchange may establish concurrent HASH_FIXED writer ownership; + * partition and bucket handling inside each writer remains delegated to the SDK. */ public class PaimonTableSink extends BaseExternalTableDataSink { private final PaimonExternalTable targetTable; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java index 558a9df8e82c4a..7da22c2f303e4c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java @@ -32,9 +32,11 @@ import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.properties.DistributionSpecExternalTableSinkHashPartitioned.WriterAssignment; +import org.apache.doris.nereids.properties.DistributionSpecExternalTableSinkUnPartitioned; import org.apache.doris.nereids.properties.DistributionSpecHiveTableSinkHashPartitioned; -import org.apache.doris.nereids.properties.DistributionSpecHiveTableSinkUnPartitioned; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; @@ -54,6 +56,7 @@ import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.ImmutableList; import mockit.Mock; import mockit.MockUp; import mockit.Mocked; @@ -528,12 +531,13 @@ public void testInsertIntoPlanSql() throws Exception { mockTargetTable(schema, new HashSet<>()); String unPartTargetTable = "unpart_" + insertTable; String insertSql = "INSERT INTO " + unPartTargetTable + " values(1, 'v1', 32.1, 'aabb')"; - PhysicalPlan physicalSink = getPhysicalPlan(insertSql, PhysicalProperties.SINK_RANDOM_PARTITIONED, - false); + PhysicalPlan physicalSink = getPhysicalPlan(insertSql, + PhysicalProperties.EXTERNAL_TABLE_SINK_UNPARTITIONED, false); checkUnpartTableSinkPlan(schema, unPartTargetTable, physicalSink); String insertOverwriteSql = "INSERT OVERWRITE TABLE " + unPartTargetTable + " values(1, 'v1', 32.1, 'aabb')"; - PhysicalPlan physicalOverwriteSink = getPhysicalPlan(insertOverwriteSql, PhysicalProperties.SINK_RANDOM_PARTITIONED, + PhysicalPlan physicalOverwriteSink = getPhysicalPlan(insertOverwriteSql, + PhysicalProperties.EXTERNAL_TABLE_SINK_UNPARTITIONED, true); checkUnpartTableSinkPlan(schema, unPartTargetTable, physicalOverwriteSink); @@ -558,12 +562,12 @@ public void testInsertIntoPlanSql() throws Exception { String insertSql2 = "INSERT INTO " + partTargetTable + " values(1, 'v1', 'v2', '2020-03-13')"; PhysicalPlan physicalSink2 = getPhysicalPlan(insertSql2, - new PhysicalProperties(new DistributionSpecHiveTableSinkHashPartitioned()), false); + externalHashProperties(), false); checkPartTableSinkPlan(schema, partTargetTable, physicalSink2); String insertOverwrite2 = "INSERT OVERWRITE TABLE " + partTargetTable + " values(1, 'v1', 'v2', '2020-03-13')"; PhysicalPlan physicalOverwriteSink2 = getPhysicalPlan(insertOverwrite2, - new PhysicalProperties(new DistributionSpecHiveTableSinkHashPartitioned()), true); + externalHashProperties(), true); checkPartTableSinkPlan(schema, partTargetTable, physicalOverwriteSink2); } @@ -571,7 +575,8 @@ private static void checkUnpartTableSinkPlan(List schema, String unPartT Assertions.assertSame(physicalSink.getType(), PlanType.PHYSICAL_DISTRIBUTE); // check exchange PhysicalDistribute distribute = (PhysicalDistribute) physicalSink; - Assertions.assertTrue(distribute.getDistributionSpec() instanceof DistributionSpecHiveTableSinkUnPartitioned); + Assertions.assertTrue(distribute.getDistributionSpec() + instanceof DistributionSpecExternalTableSinkUnPartitioned); Assertions.assertSame(distribute.child(0).getType(), PlanType.PHYSICAL_HIVE_TABLE_SINK); // check sink PhysicalHiveTableSink physicalHiveSink = (PhysicalHiveTableSink) physicalSink.child(0); @@ -583,7 +588,11 @@ private static void checkPartTableSinkPlan(List schema, String unPartTar Assertions.assertSame(physicalSink.getType(), PlanType.PHYSICAL_DISTRIBUTE); // check exchange PhysicalDistribute distribute2 = (PhysicalDistribute) physicalSink; - Assertions.assertTrue(distribute2.getDistributionSpec() instanceof DistributionSpecHiveTableSinkHashPartitioned); + Assertions.assertTrue(distribute2.getDistributionSpec() + instanceof DistributionSpecHiveTableSinkHashPartitioned); + DistributionSpecHiveTableSinkHashPartitioned distributionSpec = + (DistributionSpecHiveTableSinkHashPartitioned) distribute2.getDistributionSpec(); + Assertions.assertEquals(WriterAssignment.SKEWED, distributionSpec.getWriterAssignment()); Assertions.assertSame(distribute2.child(0).getType(), PlanType.PHYSICAL_HIVE_TABLE_SINK); // check sink PhysicalHiveTableSink physicalHiveSink2 = (PhysicalHiveTableSink) physicalSink.child(0); @@ -591,6 +600,11 @@ private static void checkPartTableSinkPlan(List schema, String unPartTar Assertions.assertEquals(schema.size(), physicalHiveSink2.getOutput().size()); } + private static PhysicalProperties externalHashProperties() { + return new PhysicalProperties(new DistributionSpecHiveTableSinkHashPartitioned( + ImmutableList.of(new ExprId(1)))); + } + private void createTargetTable(String tableName) throws Exception { String createInsertTable = "CREATE TABLE `unpart_" + tableName + "`(\n" + " `col1` INT COMMENT 'col1',\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index af3be4475f71ac..39ae70bf5c7559 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -27,6 +27,8 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.properties.DistributionSpecExternalTableSinkHashPartitioned.WriterAssignment; +import org.apache.doris.nereids.properties.DistributionSpecIcebergTableSinkHashPartitioned; import org.apache.doris.nereids.properties.DistributionSpecMerge; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Alias; @@ -40,6 +42,7 @@ import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; import org.apache.doris.nereids.trees.plans.commands.UpdateCommand; import org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.commands.merge.MergeIntoCommand; import org.apache.doris.nereids.trees.plans.commands.use.SwitchCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergDeleteSink; @@ -49,6 +52,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergDeleteSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergMergeSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.util.MemoTestUtils; @@ -62,6 +66,7 @@ import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.RowLevelOperationMode; import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; @@ -183,6 +188,7 @@ protected void runBeforeAll() throws Exception { Mockito.doReturn(mockedSpec).when(mockedIcebergTable).spec(); Mockito.doReturn(ImmutableMap.of()).when(mockedIcebergTable).specs(); Mockito.doReturn(icebergSchema).when(mockedIcebergTable).schema(); + Mockito.doReturn(SortOrder.unsorted()).when(mockedIcebergTable).sortOrder(); IcebergSnapshotCacheValue snapshotCacheValue = new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), new IcebergSnapshot(0L, 0L), Optional.empty(), mockedIcebergTable); Mockito.doReturn(new IcebergMvccSnapshot(snapshotCacheValue)).when(spyTable) @@ -339,6 +345,67 @@ public void testCreateIcebergDefaultV3TableRejectsRowLineageReservedColumn() thr } } + @Test + public void testIcebergInsertUsesTransformOwnershipExchange() throws Exception { + useIceberg(); + PartitionSpec partitionSpec = PartitionSpec.builderFor(baseIcebergSchema).bucket("id", 16).build(); + Mockito.doReturn(partitionSpec).when(mockedIcebergTable).spec(); + try { + String sql = "insert into " + tableName + " values (1, 'name', 2, 3, 4.00)"; + LogicalPlan command = parseStmt(sql); + Assertions.assertTrue(command instanceof InsertIntoTableCommand); + Plan explainPlan = ((InsertIntoTableCommand) command).getExplainPlan(connectContext); + PhysicalPlan physicalPlan = planPhysicalPlan( + (LogicalPlan) explainPlan, PhysicalProperties.ANY, sql); + + PhysicalIcebergTableSink sink = + getSinglePhysicalSink(physicalPlan, PhysicalIcebergTableSink.class); + Assertions.assertTrue(sink.getRequirePhysicalProperties().getDistributionSpec() + instanceof DistributionSpecIcebergTableSinkHashPartitioned); + DistributionSpecIcebergTableSinkHashPartitioned distributionSpec = + (DistributionSpecIcebergTableSinkHashPartitioned) sink + .getRequirePhysicalProperties().getDistributionSpec(); + Assertions.assertEquals(ImmutableList.of("bucket[16]"), + distributionSpec.getPartitionTransforms()); + Assertions.assertEquals(WriterAssignment.SKEWED, distributionSpec.getWriterAssignment()); + + Set distributes = physicalPlan.collect(PhysicalDistribute.class::isInstance); + Assertions.assertTrue(distributes.stream().anyMatch(distribute -> distribute.getDistributionSpec() + instanceof DistributionSpecIcebergTableSinkHashPartitioned)); + } finally { + Mockito.doReturn(basePartitionSpec).when(mockedIcebergTable).spec(); + } + } + + @Test + public void testIcebergInsertWithReorderedColumnsUsesTransformSourceColumn() throws Exception { + useIceberg(); + PartitionSpec partitionSpec = PartitionSpec.builderFor(baseIcebergSchema).bucket("id", 16).build(); + Mockito.doReturn(partitionSpec).when(mockedIcebergTable).spec(); + try { + String sql = "insert into " + tableName + + " (name, id, age, score, amount) values ('name', 1, 2, 3, 4.00)"; + LogicalPlan command = parseStmt(sql); + Assertions.assertTrue(command instanceof InsertIntoTableCommand); + Plan explainPlan = ((InsertIntoTableCommand) command).getExplainPlan(connectContext); + PhysicalPlan physicalPlan = planPhysicalPlan( + (LogicalPlan) explainPlan, PhysicalProperties.ANY, sql); + + PhysicalIcebergTableSink sink = + getSinglePhysicalSink(physicalPlan, PhysicalIcebergTableSink.class); + DistributionSpecIcebergTableSinkHashPartitioned distributionSpec = + (DistributionSpecIcebergTableSinkHashPartitioned) sink + .getRequirePhysicalProperties().getDistributionSpec(); + ExprId idExprId = findExprIdByName(sink.child().getOutput(), "id"); + Assertions.assertEquals(ImmutableList.of(idExprId), + distributionSpec.getOutputColumnExprIds()); + Assertions.assertEquals(ImmutableList.of("bucket[16]"), + distributionSpec.getPartitionTransforms()); + } finally { + Mockito.doReturn(basePartitionSpec).when(mockedIcebergTable).spec(); + } + } + @Test public void testIcebergV3CtasRejectsRowLineageReservedColumn() throws Exception { useIceberg(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSinkTest.java index 842428d79047bb..41bd5555943a51 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSinkTest.java @@ -17,14 +17,29 @@ package org.apache.doris.nereids.trees.plans.physical; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.nereids.properties.DistributionSpecPaimonTableSinkHashPartitioned; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.StringType; + +import com.google.common.collect.ImmutableList; import org.apache.paimon.CoreOptions; +import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.VarCharType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,6 +65,136 @@ void testFixedPrimaryKeyAndDynamicModesRequireSingleWriter() { Assertions.assertTrue(PhysicalPaimonTableSink.requiresSingleWriter( table(BucketMode.KEY_DYNAMIC, Collections.singletonList("id"), Collections.emptyMap()))); + Assertions.assertFalse(PhysicalPaimonTableSink.requiresSingleWriter( + table(BucketMode.BUCKET_UNAWARE, Collections.emptyList(), + Collections.emptyMap()))); + } + + @Test + void testHashDynamicAlwaysRequiresSingleWriter() { + // HASH_DYNAMIC has no concurrent route in this change. Its SDK assigner option and + // route-field capabilities are deliberately not used to bypass GATHER. + Assertions.assertTrue(PhysicalPaimonTableSink.requiresSingleWriter( + table(BucketMode.HASH_DYNAMIC, Collections.singletonList("id"), + Collections.singletonMap( + CoreOptions.DYNAMIC_BUCKET_ASSIGNER_PARALLELISM.key(), "4")))); + Assertions.assertTrue(PhysicalPaimonTableSink.requiresSingleWriter( + table(BucketMode.HASH_DYNAMIC, Collections.singletonList("id"), + Collections.emptyMap()))); + } + + @Test + void testSupportedFixedBucketBuildsPaimonIdentityDistribution() { + FileStoreTable table = fixedBucketTable( + ImmutableList.of( + new DataField(0, "id", new IntType()), + new DataField(1, "part", new VarCharType())), + Collections.singletonList("part"), Collections.emptyMap()); + Slot id = new SlotReference("id", IntegerType.INSTANCE); + Slot part = new SlotReference("part", StringType.INSTANCE); + + DistributionSpecPaimonTableSinkHashPartitioned spec + = PhysicalPaimonTableSink.buildFixedBucketDistributionSpec( + table, + ImmutableList.of( + new Column("id", PrimitiveType.INT), + new Column("part", PrimitiveType.STRING)), + ImmutableList.of(id, part)); + + Assertions.assertNotNull(spec); + Assertions.assertEquals(4, spec.getNumBuckets()); + Assertions.assertEquals(ImmutableList.of(part.getExprId(), id.getExprId()), + spec.getOutputColumnExprIds()); + Assertions.assertEquals(Collections.singletonList(0), spec.getPartitionFieldIndexes()); + Assertions.assertEquals(Collections.singletonList(1), spec.getBucketFieldIndexes()); + Assertions.assertEquals( + DistributionSpecPaimonTableSinkHashPartitioned.WriterAssignment.IDENTITY, + spec.getWriterAssignment()); + } + + @Test + void testUnsupportedFixedBucketRouteFallsBack() { + FileStoreTable unsupportedType = fixedBucketTable( + Collections.singletonList( + new DataField(0, "id", new ArrayType(new IntType()))), + Collections.emptyList(), Collections.emptyMap()); + Assertions.assertNull(PhysicalPaimonTableSink.buildFixedBucketDistributionSpec( + unsupportedType, + Collections.singletonList(new Column("id", PrimitiveType.INT)), + Collections.singletonList(new SlotReference("id", IntegerType.INSTANCE)))); + Assertions.assertTrue(PhysicalPaimonTableSink.requiresSingleWriter(unsupportedType)); + + FileStoreTable customBucket = fixedBucketTable( + Collections.singletonList(new DataField(0, "id", new IntType())), + Collections.emptyList(), + Collections.singletonMap(CoreOptions.BUCKET_FUNCTION_TYPE.key(), "mod")); + Assertions.assertNull(PhysicalPaimonTableSink.buildFixedBucketDistributionSpec( + customBucket, + Collections.singletonList(new Column("id", PrimitiveType.INT)), + Collections.singletonList(new SlotReference("id", IntegerType.INSTANCE)))); + Assertions.assertTrue(PhysicalPaimonTableSink.requiresSingleWriter(customBucket)); + + FileStoreTable unsupportedPartitionType = fixedBucketTable( + ImmutableList.of( + new DataField(0, "id", new IntType()), + new DataField(1, "part", new ArrayType(new IntType()))), + Collections.singletonList("part"), Collections.emptyMap()); + Assertions.assertNull(PhysicalPaimonTableSink.buildFixedBucketDistributionSpec( + unsupportedPartitionType, + ImmutableList.of( + new Column("id", PrimitiveType.INT), + new Column("part", org.apache.doris.nereids.types.ArrayType + .of(IntegerType.INSTANCE).toCatalogDataType())), + ImmutableList.of( + new SlotReference("id", IntegerType.INSTANCE), + new SlotReference("part", org.apache.doris.nereids.types.ArrayType + .of(IntegerType.INSTANCE))))); + Assertions.assertTrue( + PhysicalPaimonTableSink.requiresSingleWriter(unsupportedPartitionType)); + + FileStoreTable typeMismatch = fixedBucketTable( + Collections.singletonList(new DataField(0, "id", new IntType())), + Collections.emptyList(), Collections.emptyMap()); + Assertions.assertNull(PhysicalPaimonTableSink.buildFixedBucketDistributionSpec( + typeMismatch, + Collections.singletonList(new Column("id", PrimitiveType.STRING)), + Collections.singletonList(new SlotReference("id", StringType.INSTANCE)))); + Assertions.assertTrue(PhysicalPaimonTableSink.requiresSingleWriter(typeMismatch)); + + Assertions.assertNull(PhysicalPaimonTableSink.buildFixedBucketDistributionSpec( + typeMismatch, + Collections.singletonList(new Column("different_name", PrimitiveType.INT)), + Collections.singletonList(new SlotReference( + "different_name", IntegerType.INSTANCE)))); + } + + @Test + void testDefaultedFixedBucketRouteFieldsFallBack() { + FileStoreTable defaultedBucketKey = fixedBucketTable( + Collections.singletonList( + new DataField(0, "id", new IntType(), null, "1")), + Collections.emptyList(), Collections.emptyMap()); + Assertions.assertNull(PhysicalPaimonTableSink.buildFixedBucketDistributionSpec( + defaultedBucketKey, + Collections.singletonList(new Column("id", PrimitiveType.INT)), + Collections.singletonList(new SlotReference("id", IntegerType.INSTANCE)))); + Assertions.assertTrue(PhysicalPaimonTableSink.requiresSingleWriter(defaultedBucketKey)); + + FileStoreTable defaultedPartitionKey = fixedBucketTable( + ImmutableList.of( + new DataField(0, "id", new IntType()), + new DataField(1, "part", new VarCharType(VarCharType.MAX_LENGTH), + null, "default-partition")), + Collections.singletonList("part"), Collections.emptyMap()); + Assertions.assertNull(PhysicalPaimonTableSink.buildFixedBucketDistributionSpec( + defaultedPartitionKey, + ImmutableList.of( + new Column("id", PrimitiveType.INT), + new Column("part", PrimitiveType.STRING)), + ImmutableList.of( + new SlotReference("id", IntegerType.INSTANCE), + new SlotReference("part", StringType.INSTANCE)))); + Assertions.assertTrue(PhysicalPaimonTableSink.requiresSingleWriter(defaultedPartitionKey)); } private static FileStoreTable table( @@ -60,4 +205,19 @@ private static FileStoreTable table( Mockito.when(table.options()).thenReturn(options); return table; } + + private static FileStoreTable fixedBucketTable(List fields, + List partitionKeys, Map extraOptions) { + Map options = new HashMap<>(extraOptions); + options.put(CoreOptions.BUCKET.key(), "4"); + options.put(CoreOptions.BUCKET_KEY.key(), "id"); + TableSchema schema = new TableSchema( + 0L, fields, 0, partitionKeys, Collections.singletonList("id"), options, null); + FileStoreTable table = Mockito.mock(FileStoreTable.class); + Mockito.when(table.bucketMode()).thenReturn(BucketMode.HASH_FIXED); + Mockito.when(table.primaryKeys()).thenReturn(Collections.singletonList("id")); + Mockito.when(table.options()).thenReturn(options); + Mockito.when(table.schema()).thenReturn(schema); + return table; + } } diff --git a/gensrc/thrift/Partitions.thrift b/gensrc/thrift/Partitions.thrift index de6ba8127affdd..738636961157ac 100644 --- a/gensrc/thrift/Partitions.thrift +++ b/gensrc/thrift/Partitions.thrift @@ -45,11 +45,12 @@ enum TPartitionType { // used for shuffle data by parititon and tablet OLAP_TABLE_SINK_HASH_PARTITIONED = 6, - // used for shuffle data by hive parititon - HIVE_TABLE_SINK_HASH_PARTITIONED = 7, + // used for shuffle data by external table sink ownership key. BE execution + // versions before 12 reject this type because value 7 had different semantics. + EXTERNAL_TABLE_SINK_HASH_PARTITIONED = 7, - // used for hive unparititoned table - HIVE_TABLE_SINK_UNPARTITIONED = 8, + // adaptive writer distribution for external tables without an ownership key + EXTERNAL_TABLE_SINK_UNPARTITIONED = 8, // used for merge partitioning: insert by partition columns, delete by row_id MERGE_PARTITIONED = 9 @@ -109,6 +110,51 @@ struct TMergePartitionInfo { 6: optional i32 partition_spec_id } +// The routing algorithm implemented by an external table sink hash exchange. +// Connector-specific FE distribution specs map to one of these algorithms. +enum TExternalTableSinkHashAlgorithm { + DIRECT_HASH = 0, + ICEBERG_TRANSFORM = 1, + + // Paimon HASH_FIXED routing using the default bucket function and + // ChannelComputer. The partition expressions carry their Doris types. + PAIMON_FIXED_BUCKET = 2 +} + +// Maps the logical partitions produced by the hash algorithm to Doris exchange writers. +// IDENTITY preserves one-writer ownership; SKEWED retains ScaleWriter behavior for formats +// which allow a hot partition to be written by multiple writers. +enum TExternalTableSinkWriterAssignment { + IDENTITY = 0, + SKEWED = 1 +} + +// Minimal metadata for Paimon's stateless HASH_FIXED route. Field indexes refer +// to TDataPartition.partition_exprs, not to physical Block column positions. +struct TPaimonFixedBucketInfo { + 1: required i32 num_buckets + 2: required list partition_field_indexes + 3: required list bucket_field_indexes +} + +// Connector-independent routing metadata for an external table sink hash +// exchange. Algorithm-specific fields are optional for protocol evolution, but +// each algorithm validates the fields it requires before processing rows. +struct TExternalTableSinkHashPartitionInfo { + 1: required TExternalTableSinkHashAlgorithm algorithm + + // Positional Iceberg transforms. Required by ICEBERG_TRANSFORM and must have + // the same size as partition_exprs. + 2: optional list partition_transforms + + // Kept optional on the wire so a new BE can reject an old FE with a clear + // status instead of silently relying on the enum's default value. + 3: optional TExternalTableSinkWriterAssignment writer_assignment + + // Required only by PAIMON_FIXED_BUCKET. + 4: optional TPaimonFixedBucketInfo paimon_fixed_bucket_info +} + // Specification of how a single logical data stream is partitioned. // This leaves out the parameters that determine the physical partition (for hash // partitions, the number of partitions; for range partitions, the partitions' @@ -118,4 +164,5 @@ struct TDataPartition { 2: optional list partition_exprs 3: optional list partition_infos 4: optional TMergePartitionInfo merge_partition_info + 5: optional TExternalTableSinkHashPartitionInfo external_table_sink_hash_partition_info } diff --git a/regression-test/data/paimon_write/test_paimon_write_pk.out b/regression-test/data/paimon_write/test_paimon_write_pk.out index c7ba2e52a398fd..5e0eddd7312e0f 100644 --- a/regression-test/data/paimon_write/test_paimon_write_pk.out +++ b/regression-test/data/paimon_write/test_paimon_write_pk.out @@ -44,7 +44,7 @@ emoji_😀 3 emoji_payload -- !pk_writer_scaling_plan -- PhysicalPaimonTableSink ---PhysicalDistribute[DistributionSpecGather] +--PhysicalDistribute[DistributionSpecPaimonTableSinkHashPartitioned] ----PhysicalProject ------PhysicalTVFRelation diff --git a/regression-test/suites/paimon_write/test_paimon_write_pk.groovy b/regression-test/suites/paimon_write/test_paimon_write_pk.groovy index 39496118f0e420..4034cab882c411 100644 --- a/regression-test/suites/paimon_write/test_paimon_write_pk.groovy +++ b/regression-test/suites/paimon_write/test_paimon_write_pk.groovy @@ -174,8 +174,8 @@ suite("test_paimon_write_pk", "p0,external,paimon") { FROM t_pk_string_bucket ORDER BY event_id""" assertTableEquals("t_pk_string_bucket", "ORDER BY event_id") - // FT-045: A fixed-bucket PK table uses one writer until Doris has a - // bucket-aware exchange, so all versions owned by one bucket stay together. + // FT-045: The bucket-aware Exchange may use multiple writers, but every + // (partition, bucket) remains owned by exactly one writer. sql """SET parallel_pipeline_task_num = 4""" sql """SET enable_strict_consistency_dml = false""" qt_pk_writer_scaling_plan """EXPLAIN SHAPE PLAN