From 85ade0c26d91c8c066fe823c590818eeb9bd31e0 Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Wed, 28 Oct 2026 09:49:05 +0800 Subject: [PATCH 1/3] [improvement](be) Reduce transitive dependencies in shared headers ### What problem does this PR solve? Problem Summary: Shared BE headers propagate protobuf, Thrift, cache, thread-pool and storage-reader definitions into unrelated translation units, increasing repeated parsing and header rebuild scope. Introduce dedicated ColumnIteratorOptions and SyncSizeCallback headers, use forward declarations where complete types are unnecessary, and move protobuf conversions, number SerDe definitions and lifecycle operations into their implementation files. Add direct includes at actual use sites and repair independent compilation of affected production and test sources. Preserve object layouts, move semantics, explicit template instantiations and per-value decoding implementations. Across the 16 monitored heavy headers, measured header-to-TU dependency pairs decrease from 8,108 to 6,617 (18.39%), affecting 742 distinct TUs. RLE reach decreases from 537 to 154 TUs and threadpool reach from 607 to 361. The dependency.h project-header closure decreases from 325 to 246. Header budgets and exception lists remain unchanged. Release builds with Clang 21.1.8, -j50, PCH/Unity/ccache disabled measured 636.4 seconds before, 569.0 seconds after and 636.2 seconds on repetition. Concurrent machine load prevents claiming a stable full-build speedup. Median production compiler peak RSS changes from 563.5 MiB to 551.5 MiB and 551.6 MiB; production compilation remains at 1,531 translation units. ### Release note None ### Check List (For Author) - Test: Three complete Release builds; standard ASAN BE/FE build and final incremental build; all BE UT sources compiled and linked; 29 selected unit tests passed with -j50 --coverage; 1,024-row service read, aggregate, shuffle join, sort and cast checks; header hygiene and changed-line format checks passed. Clang-tidy covered all 71 files and retained two existing warnings in relocated protobuf methods (function size and signedness). - Behavior changed: No - Does this need documentation: No --- be/src/cloud/cloud_index_change_compaction.h | 1 + be/src/cloud/cloud_schema_change_job.h | 4 + be/src/cloud/cloud_tablet.h | 1 + be/src/common/signal_handler.h | 1 + .../data_type_number_serde.cpp | 170 +++++++++++++++++ .../data_type_serde/data_type_number_serde.h | 171 +----------------- .../data_type_serde/data_type_uuid_serde.cpp | 1 + be/src/exec/exchange/vdata_stream_recvr.cpp | 13 ++ be/src/exec/exchange/vdata_stream_recvr.h | 13 +- be/src/exec/pipeline/pipeline_task.cpp | 1 + be/src/exec/rowid_fetcher.cpp | 2 + .../runtime_filter_producer.cpp | 1 + .../runtime_filter/runtime_filter_producer.h | 60 +----- .../exec/runtime_filter/sync_size_callback.h | 85 +++++++++ .../function_date_or_datetime_to_string.cpp | 1 + .../function/function_datetime_floor_ceil.cpp | 1 + .../function_datetime_string_to_string.cpp | 1 + be/src/exprs/function/function_java_udf.cpp | 1 + .../function/function_other_types_to_date.cpp | 1 + .../exprs/function/function_variant_type.cpp | 1 + be/src/exprs/function/time_of_function.cpp | 1 + be/src/exprs/function/to_time_function.cpp | 1 + be/src/exprs/vexpr_context.h | 2 +- be/src/format_v2/materialized_reader_util.cpp | 1 + .../table/fluss_union_lake_reader.cpp | 1 + be/src/format_v2/table/iceberg_reader.cpp | 1 + be/src/format_v2/table_reader.h | 1 + .../schema_column_data_sizes_scanner.h | 3 +- be/src/io/cache/async_cache_write_manager.cpp | 1 + be/src/io/cache/async_cache_write_manager.h | 5 +- be/src/io/cache/block_file_cache.h | 1 - be/src/io/cache/block_file_cache_factory.cpp | 2 + be/src/io/cache/block_file_cache_factory.h | 5 +- be/src/io/cache/file_cache_common.cpp | 1 + be/src/io/fs/hdfs_file_writer.cpp | 1 + be/src/io/fs/packed_file_manager.h | 5 +- be/src/io/fs/s3_file_reader.cpp | 1 + be/src/io/fs/s3_file_reader.h | 2 +- be/src/io/fs/stream_sink_file_writer.h | 1 + be/src/io/hdfs_util.h | 2 +- be/src/runtime/query_context.h | 1 + be/src/runtime/runtime_state.cpp | 14 ++ be/src/runtime/runtime_state.h | 17 +- .../workload_group/workload_group_manager.h | 9 +- be/src/service/http/http_channel.cpp | 3 + be/src/storage/id_manager.h | 3 +- .../storage/segment/column_iterator_options.h | 52 ++++++ be/src/storage/segment/column_reader.h | 20 +- be/src/storage/tablet/tablet_manager.cpp | 3 +- be/src/storage/tablet/tablet_manager.h | 7 +- be/src/storage/tablet/tablet_meta.h | 1 - be/src/util/s3_util.cpp | 1 + be/src/util/s3_util.h | 5 +- be/src/util/time.cpp | 6 +- be/src/util/uid_util.cpp | 44 +++++ be/src/util/uid_util.h | 47 ++--- .../cloud/cloud_schema_change_job_test.cpp | 1 + be/test/exec/pipeline/pipeline_task_test.cpp | 2 + .../sync_size_callback_test.cpp | 2 + be/test/exprs/index_exec_context_test.cpp | 1 + .../vexpr_evalute_inverted_index_test.cpp | 1 + .../format_v2/parquet/native_decoder_test.cpp | 15 +- be/test/format_v2/table/hudi_reader_test.cpp | 1 + be/test/io/client/s3_file_system_test.cpp | 1 + ...fragment_mgr_cross_cluster_cancel_test.cpp | 1 + .../memory/thread_mem_tracker_mgr_test.cpp | 1 + .../cloud_index_change_task_test.cpp | 3 +- be/test/util/uid_util_test.cpp | 2 + 68 files changed, 499 insertions(+), 333 deletions(-) create mode 100644 be/src/exec/runtime_filter/sync_size_callback.h create mode 100644 be/src/storage/segment/column_iterator_options.h diff --git a/be/src/cloud/cloud_index_change_compaction.h b/be/src/cloud/cloud_index_change_compaction.h index e0bd7952ca7568..afd028cf1eda90 100644 --- a/be/src/cloud/cloud_index_change_compaction.h +++ b/be/src/cloud/cloud_index_change_compaction.h @@ -21,6 +21,7 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" +#include "gen_cpp/cloud.pb.h" #include "storage/compaction/compaction.h" namespace doris { diff --git a/be/src/cloud/cloud_schema_change_job.h b/be/src/cloud/cloud_schema_change_job.h index 8428e17c496ced..0f5b039c83b83d 100644 --- a/be/src/cloud/cloud_schema_change_job.h +++ b/be/src/cloud/cloud_schema_change_job.h @@ -27,6 +27,10 @@ namespace doris { +namespace cloud { +class TabletJobInfoPB; +} + class CloudSchemaChangeJob { public: CloudSchemaChangeJob(CloudStorageEngine& cloud_storage_engine, std::string job_id, diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index 9743a40960e3bb..28675912bab494 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -26,6 +26,7 @@ #include "storage/tablet/base_tablet.h" namespace doris { +class TTabletInfo; class CloudStorageEngine; diff --git a/be/src/common/signal_handler.h b/be/src/common/signal_handler.h index 334b2ea99af6ce..47f30503f17ce4 100644 --- a/be/src/common/signal_handler.h +++ b/be/src/common/signal_handler.h @@ -34,6 +34,7 @@ #pragma once #include // ARRAYSIZE_UNSAFE +#include #include #include diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 1ce576b28134de..9eab9b00dbb11d 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -2005,6 +2006,175 @@ Status DataTypeNumberSerDe::read_column_from_orc(IColumn& column, return DataTypeSerDe::read_column_from_orc(column, view); } +template +Status DataTypeNumberSerDe::read_column_from_pb(IColumn& column, const PValues& arg) const { + auto old_column_size = column.size(); + if constexpr (T == TYPE_BOOLEAN) { + column.resize(old_column_size + arg.uint32_value_size()); + auto& data = assert_cast(column).get_data(); + for (int i = 0; i < arg.uint32_value_size(); ++i) { + data[old_column_size + i] = + cast_set::CppType, uint32_t, false>( + arg.uint32_value(i)); + } + } else if constexpr (T == TYPE_DATEV2 || T == TYPE_IPV4) { + column.resize(old_column_size + arg.uint32_value_size()); + auto& data = assert_cast(column).get_data(); + for (int i = 0; i < arg.uint32_value_size(); ++i) { + data[old_column_size + i] = arg.uint32_value(i); + } + } else if constexpr (T == TYPE_TINYINT || T == TYPE_SMALLINT) { + column.resize(old_column_size + arg.int32_value_size()); + auto& data = reinterpret_cast(column).get_data(); + for (int i = 0; i < arg.int32_value_size(); ++i) { + data[old_column_size + i] = + cast_set::CppType, int32_t, false>( + arg.int32_value(i)); + } + } else if constexpr (T == TYPE_INT) { + column.resize(old_column_size + arg.int32_value_size()); + auto& data = reinterpret_cast(column).get_data(); + for (int i = 0; i < arg.int32_value_size(); ++i) { + data[old_column_size + i] = arg.int32_value(i); + } + } else if constexpr (T == TYPE_DATETIMEV2) { + column.resize(old_column_size + arg.uint64_value_size()); + auto& data = reinterpret_cast(column).get_data(); + for (int i = 0; i < arg.uint64_value_size(); ++i) { + data[old_column_size + i] = + binary_cast>(arg.uint64_value(i)); + } + } else if constexpr (T == TYPE_TIMESTAMPTZ) { + column.resize(old_column_size + arg.uint64_value_size()); + auto& data = reinterpret_cast(column).get_data(); + for (int i = 0; i < arg.uint64_value_size(); ++i) { + data[old_column_size + i] = binary_cast(arg.uint64_value(i)); + } + } else if constexpr (T == TYPE_DATE || T == TYPE_DATETIME) { + column.resize(old_column_size + arg.int64_value_size()); + auto& data = reinterpret_cast(column).get_data(); + for (int i = 0; i < arg.int64_value_size(); ++i) { + data[old_column_size + i] = binary_cast(arg.int64_value(i)); + } + } else if constexpr (T == TYPE_BIGINT) { + column.resize(old_column_size + arg.int64_value_size()); + auto& data = reinterpret_cast(column).get_data(); + for (int i = 0; i < arg.int64_value_size(); ++i) { + data[old_column_size + i] = arg.int64_value(i); + } + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + column.resize(old_column_size + arg.int64_value_size()); + auto& data = reinterpret_cast(column).get_data(); + for (int i = 0; i < arg.int64_value_size(); ++i) { + data[old_column_size + i] = TimeStampNsValue(arg.int64_value(i)); + } + } else if constexpr (T == TYPE_FLOAT) { + column.resize(old_column_size + arg.float_value_size()); + auto& data = reinterpret_cast(column).get_data(); + for (int i = 0; i < arg.float_value_size(); ++i) { + data[old_column_size + i] = arg.float_value(i); + } + } else if constexpr (T == TYPE_DOUBLE || T == TYPE_TIMEV2) { + column.resize(old_column_size + arg.double_value_size()); + auto& data = reinterpret_cast(column).get_data(); + for (int i = 0; i < arg.double_value_size(); ++i) { + data[old_column_size + i] = arg.double_value(i); + } + } else if constexpr (T == TYPE_LARGEINT) { + column.resize(old_column_size + arg.bytes_value_size()); + auto& data = reinterpret_cast(column).get_data(); + for (int i = 0; i < arg.bytes_value_size(); ++i) { + data[old_column_size + i] = *(int128_t*)(arg.bytes_value(i).c_str()); + } + } else { + return Status::NotSupported("unknown ColumnType for reading from pb"); + } + return Status::OK(); +} + +template +Status DataTypeNumberSerDe::write_column_to_pb(const IColumn& column, PValues& result, + int64_t start, int64_t end) const { + auto row_count = cast_set(end - start); + auto* ptype = result.mutable_type(); + const auto* col = assert_cast(&column); + if constexpr (T == TYPE_LARGEINT) { + ptype->set_id(PGenericType::INT128); + result.mutable_bytes_value()->Reserve(row_count); + for (size_t row_num = start; row_num < end; ++row_num) { + StringRef single_data = col->get_data_at(row_num); + result.add_bytes_value(single_data.data, single_data.size); + } + return Status::OK(); + } + auto& data = col->get_data(); + if constexpr (T == TYPE_BOOLEAN) { + ptype->set_id(PGenericType::UINT8); + auto* values = result.mutable_uint32_value(); + values->Reserve(row_count); + values->Add(data.begin() + start, data.begin() + end); + } else if constexpr (T == TYPE_DATEV2) { + ptype->set_id(PGenericType::UINT32); + auto* values = result.mutable_uint32_value(); + values->Reserve(row_count); + values->Add((uint32_t*)data.begin() + start, (uint32_t*)data.begin() + end); + } else if constexpr (T == TYPE_IPV4) { + ptype->set_id(PGenericType::UINT32); + auto* values = result.mutable_uint32_value(); + values->Reserve(row_count); + values->Add(data.begin() + start, data.begin() + end); + } else if constexpr (T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMPTZ) { + ptype->set_id(PGenericType::UINT64); + auto* values = result.mutable_uint64_value(); + values->Reserve(row_count); + values->Add((uint64_t*)data.begin() + start, (uint64_t*)data.begin() + end); + } else if constexpr (T == TYPE_TINYINT) { + ptype->set_id(PGenericType::INT8); + auto* values = result.mutable_int32_value(); + values->Reserve(row_count); + values->Add(data.begin() + start, data.begin() + end); + } else if constexpr (T == TYPE_SMALLINT) { + ptype->set_id(PGenericType::INT16); + auto* values = result.mutable_int32_value(); + values->Reserve(row_count); + values->Add(data.begin() + start, data.begin() + end); + } else if constexpr (T == TYPE_INT) { + ptype->set_id(PGenericType::INT32); + auto* values = result.mutable_int32_value(); + values->Reserve(row_count); + values->Add(data.begin() + start, data.begin() + end); + } else if constexpr (T == TYPE_DATE || T == TYPE_DATETIME) { + ptype->set_id(PGenericType::INT64); + auto* values = result.mutable_int64_value(); + values->Reserve(row_count); + values->Add((int64_t*)data.begin() + start, (int64_t*)data.begin() + end); + } else if constexpr (T == TYPE_BIGINT) { + ptype->set_id(PGenericType::INT64); + auto* values = result.mutable_int64_value(); + values->Reserve(row_count); + values->Add(data.begin() + start, data.begin() + end); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + ptype->set_id(PGenericType::INT64); + auto* values = result.mutable_int64_value(); + values->Reserve(row_count); + values->Add(reinterpret_cast(data.begin()) + start, + reinterpret_cast(data.begin()) + end); + } else if constexpr (T == TYPE_FLOAT) { + ptype->set_id(PGenericType::FLOAT); + auto* values = result.mutable_float_value(); + values->Reserve(row_count); + values->Add(data.begin() + start, data.begin() + end); + } else if constexpr (T == TYPE_DOUBLE || T == TYPE_TIMEV2) { + ptype->set_id(PGenericType::DOUBLE); + auto* values = result.mutable_double_value(); + values->Reserve(row_count); + values->Add(data.begin() + start, data.begin() + end); + } else { + return Status::NotSupported("unknown ColumnType for writing to pb"); + } + return Status::OK(); +} + /// Explicit template instantiations - to avoid code bloat in headers. template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index d067ae720a9866..47674c48cc5750 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include @@ -37,6 +36,7 @@ namespace doris { class JsonbOutStream; class Arena; +class PValues; // special data type using, maybe has various serde actions, so use specific date serde // DataTypeDateV2 => T:UInt32 @@ -194,175 +194,6 @@ class DataTypeNumberSerDe : public DataTypeSerDe { mutable IColumn::Filter _parquet_predicate_nulls; }; -template -Status DataTypeNumberSerDe::read_column_from_pb(IColumn& column, const PValues& arg) const { - auto old_column_size = column.size(); - if constexpr (T == TYPE_BOOLEAN) { - column.resize(old_column_size + arg.uint32_value_size()); - auto& data = assert_cast(column).get_data(); - for (int i = 0; i < arg.uint32_value_size(); ++i) { - data[old_column_size + i] = - cast_set::CppType, uint32_t, false>( - arg.uint32_value(i)); - } - } else if constexpr (T == TYPE_DATEV2 || T == TYPE_IPV4) { - column.resize(old_column_size + arg.uint32_value_size()); - auto& data = assert_cast(column).get_data(); - for (int i = 0; i < arg.uint32_value_size(); ++i) { - data[old_column_size + i] = arg.uint32_value(i); - } - } else if constexpr (T == TYPE_TINYINT || T == TYPE_SMALLINT) { - column.resize(old_column_size + arg.int32_value_size()); - auto& data = reinterpret_cast(column).get_data(); - for (int i = 0; i < arg.int32_value_size(); ++i) { - data[old_column_size + i] = - cast_set::CppType, int32_t, false>( - arg.int32_value(i)); - } - } else if constexpr (T == TYPE_INT) { - column.resize(old_column_size + arg.int32_value_size()); - auto& data = reinterpret_cast(column).get_data(); - for (int i = 0; i < arg.int32_value_size(); ++i) { - data[old_column_size + i] = arg.int32_value(i); - } - } else if constexpr (T == TYPE_DATETIMEV2) { - column.resize(old_column_size + arg.uint64_value_size()); - auto& data = reinterpret_cast(column).get_data(); - for (int i = 0; i < arg.uint64_value_size(); ++i) { - data[old_column_size + i] = - binary_cast>(arg.uint64_value(i)); - } - } else if constexpr (T == TYPE_TIMESTAMPTZ) { - column.resize(old_column_size + arg.uint64_value_size()); - auto& data = reinterpret_cast(column).get_data(); - for (int i = 0; i < arg.uint64_value_size(); ++i) { - data[old_column_size + i] = binary_cast(arg.uint64_value(i)); - } - } else if constexpr (T == TYPE_DATE || T == TYPE_DATETIME) { - column.resize(old_column_size + arg.int64_value_size()); - auto& data = reinterpret_cast(column).get_data(); - for (int i = 0; i < arg.int64_value_size(); ++i) { - data[old_column_size + i] = binary_cast(arg.int64_value(i)); - } - } else if constexpr (T == TYPE_BIGINT) { - column.resize(old_column_size + arg.int64_value_size()); - auto& data = reinterpret_cast(column).get_data(); - for (int i = 0; i < arg.int64_value_size(); ++i) { - data[old_column_size + i] = arg.int64_value(i); - } - } else if constexpr (T == TYPE_TIMESTAMP_NS) { - column.resize(old_column_size + arg.int64_value_size()); - auto& data = reinterpret_cast(column).get_data(); - for (int i = 0; i < arg.int64_value_size(); ++i) { - data[old_column_size + i] = TimeStampNsValue(arg.int64_value(i)); - } - } else if constexpr (T == TYPE_FLOAT) { - column.resize(old_column_size + arg.float_value_size()); - auto& data = reinterpret_cast(column).get_data(); - for (int i = 0; i < arg.float_value_size(); ++i) { - data[old_column_size + i] = arg.float_value(i); - } - } else if constexpr (T == TYPE_DOUBLE || T == TYPE_TIMEV2) { - column.resize(old_column_size + arg.double_value_size()); - auto& data = reinterpret_cast(column).get_data(); - for (int i = 0; i < arg.double_value_size(); ++i) { - data[old_column_size + i] = arg.double_value(i); - } - } else if constexpr (T == TYPE_LARGEINT) { - column.resize(old_column_size + arg.bytes_value_size()); - auto& data = reinterpret_cast(column).get_data(); - for (int i = 0; i < arg.bytes_value_size(); ++i) { - data[old_column_size + i] = *(int128_t*)(arg.bytes_value(i).c_str()); - } - } else { - return Status::NotSupported("unknown ColumnType for reading from pb"); - } - return Status::OK(); -} - -template -Status DataTypeNumberSerDe::write_column_to_pb(const IColumn& column, PValues& result, - int64_t start, int64_t end) const { - auto row_count = cast_set(end - start); - auto* ptype = result.mutable_type(); - const auto* col = assert_cast(&column); - if constexpr (T == TYPE_LARGEINT) { - ptype->set_id(PGenericType::INT128); - result.mutable_bytes_value()->Reserve(row_count); - for (size_t row_num = start; row_num < end; ++row_num) { - StringRef single_data = col->get_data_at(row_num); - result.add_bytes_value(single_data.data, single_data.size); - } - return Status::OK(); - } - auto& data = col->get_data(); - if constexpr (T == TYPE_BOOLEAN) { - ptype->set_id(PGenericType::UINT8); - auto* values = result.mutable_uint32_value(); - values->Reserve(row_count); - values->Add(data.begin() + start, data.begin() + end); - } else if constexpr (T == TYPE_DATEV2) { - ptype->set_id(PGenericType::UINT32); - auto* values = result.mutable_uint32_value(); - values->Reserve(row_count); - values->Add((uint32_t*)data.begin() + start, (uint32_t*)data.begin() + end); - } else if constexpr (T == TYPE_IPV4) { - ptype->set_id(PGenericType::UINT32); - auto* values = result.mutable_uint32_value(); - values->Reserve(row_count); - values->Add(data.begin() + start, data.begin() + end); - } else if constexpr (T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMPTZ) { - ptype->set_id(PGenericType::UINT64); - auto* values = result.mutable_uint64_value(); - values->Reserve(row_count); - values->Add((uint64_t*)data.begin() + start, (uint64_t*)data.begin() + end); - } else if constexpr (T == TYPE_TINYINT) { - ptype->set_id(PGenericType::INT8); - auto* values = result.mutable_int32_value(); - values->Reserve(row_count); - values->Add(data.begin() + start, data.begin() + end); - } else if constexpr (T == TYPE_SMALLINT) { - ptype->set_id(PGenericType::INT16); - auto* values = result.mutable_int32_value(); - values->Reserve(row_count); - values->Add(data.begin() + start, data.begin() + end); - } else if constexpr (T == TYPE_INT) { - ptype->set_id(PGenericType::INT32); - auto* values = result.mutable_int32_value(); - values->Reserve(row_count); - values->Add(data.begin() + start, data.begin() + end); - } else if constexpr (T == TYPE_DATE || T == TYPE_DATETIME) { - ptype->set_id(PGenericType::INT64); - auto* values = result.mutable_int64_value(); - values->Reserve(row_count); - values->Add((int64_t*)data.begin() + start, (int64_t*)data.begin() + end); - } else if constexpr (T == TYPE_BIGINT) { - ptype->set_id(PGenericType::INT64); - auto* values = result.mutable_int64_value(); - values->Reserve(row_count); - values->Add(data.begin() + start, data.begin() + end); - } else if constexpr (T == TYPE_TIMESTAMP_NS) { - ptype->set_id(PGenericType::INT64); - auto* values = result.mutable_int64_value(); - values->Reserve(row_count); - values->Add(reinterpret_cast(data.begin()) + start, - reinterpret_cast(data.begin()) + end); - } else if constexpr (T == TYPE_FLOAT) { - ptype->set_id(PGenericType::FLOAT); - auto* values = result.mutable_float_value(); - values->Reserve(row_count); - values->Add(data.begin() + start, data.begin() + end); - } else if constexpr (T == TYPE_DOUBLE || T == TYPE_TIMEV2) { - ptype->set_id(PGenericType::DOUBLE); - auto* values = result.mutable_double_value(); - values->Reserve(row_count); - values->Add(data.begin() + start, data.begin() + end); - } else { - return Status::NotSupported("unknown ColumnType for writing to pb"); - } - return Status::OK(); -} - template <> Status DataTypeNumberSerDe::write_column_to_arrow( const IColumn& column, const NullMap* null_map, arrow::ArrayBuilder* array_builder, diff --git a/be/src/core/data_type_serde/data_type_uuid_serde.cpp b/be/src/core/data_type_serde/data_type_uuid_serde.cpp index 19a87970153d6a..16ad36d3521949 100644 --- a/be/src/core/data_type_serde/data_type_uuid_serde.cpp +++ b/be/src/core/data_type_serde/data_type_uuid_serde.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include diff --git a/be/src/exec/exchange/vdata_stream_recvr.cpp b/be/src/exec/exchange/vdata_stream_recvr.cpp index f248549e222988..ec606c6f79cbe7 100644 --- a/be/src/exec/exchange/vdata_stream_recvr.cpp +++ b/be/src/exec/exchange/vdata_stream_recvr.cpp @@ -41,6 +41,19 @@ #include "util/uid_util.h" namespace doris { +VDataStreamRecvr::SenderQueue::BlockItem::BlockItem() = default; + +VDataStreamRecvr::SenderQueue::BlockItem::BlockItem(BlockUPtr&& block, size_t block_byte_size) + : _block(std::move(block)), _block_byte_size(block_byte_size) {} + +VDataStreamRecvr::SenderQueue::BlockItem::BlockItem(std::unique_ptr&& pblock, + size_t block_byte_size) + : _block(nullptr), _pblock(std::move(pblock)), _block_byte_size(block_byte_size) {} + +VDataStreamRecvr::SenderQueue::BlockItem::BlockItem(BlockItem&&) noexcept = default; +VDataStreamRecvr::SenderQueue::BlockItem& VDataStreamRecvr::SenderQueue::BlockItem::operator=( + BlockItem&&) noexcept = default; +VDataStreamRecvr::SenderQueue::BlockItem::~BlockItem() = default; VDataStreamRecvr::SenderQueue::SenderQueue(VDataStreamRecvr* parent_recvr, int num_senders, std::shared_ptr local_channel_dependency) diff --git a/be/src/exec/exchange/vdata_stream_recvr.h b/be/src/exec/exchange/vdata_stream_recvr.h index da139218b6b59b..3c4c6940f3821a 100644 --- a/be/src/exec/exchange/vdata_stream_recvr.h +++ b/be/src/exec/exchange/vdata_stream_recvr.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include #include @@ -258,12 +257,12 @@ class VDataStreamRecvr::SenderQueue { int64_t deserialize_time() const { return _deserialize_time; } int64_t decompress_time() const { return _decompress_time; } size_t decompress_bytes() const { return _decompress_bytes; } - BlockItem() = default; - BlockItem(BlockUPtr&& block, size_t block_byte_size) - : _block(std::move(block)), _block_byte_size(block_byte_size) {} - - BlockItem(std::unique_ptr&& pblock, size_t block_byte_size) - : _block(nullptr), _pblock(std::move(pblock)), _block_byte_size(block_byte_size) {} + BlockItem(); + BlockItem(BlockUPtr&& block, size_t block_byte_size); + BlockItem(std::unique_ptr&& pblock, size_t block_byte_size); + BlockItem(BlockItem&&) noexcept; + BlockItem& operator=(BlockItem&&) noexcept; + ~BlockItem(); void set_done(google::protobuf::Closure* done) { // The done callback is only set when the queue memory limit is exceeded. diff --git a/be/src/exec/pipeline/pipeline_task.cpp b/be/src/exec/pipeline/pipeline_task.cpp index e614fa92816189..d50af9e12b5786 100644 --- a/be/src/exec/pipeline/pipeline_task.cpp +++ b/be/src/exec/pipeline/pipeline_task.cpp @@ -48,6 +48,7 @@ #include "runtime/runtime_profile.h" #include "runtime/runtime_profile_counter_names.h" #include "runtime/thread_context.h" +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_group/workload_group_manager.h" #include "util/defer_op.h" #include "util/mem_info.h" diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp index 2c3537e755a103..d240624e3f8d8a 100644 --- a/be/src/exec/rowid_fetcher.cpp +++ b/be/src/exec/rowid_fetcher.cpp @@ -48,8 +48,10 @@ #include "runtime/exec_env.h" // ExecEnv #include "runtime/fragment_mgr.h" // FragmentMgr #include "runtime/runtime_state.h" // RuntimeState +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_group/workload_group_manager.h" #include "semaphore" +#include "service/backend_options.h" #include "storage/olap_common.h" #include "storage/rowset/beta_rowset.h" #include "storage/segment/column_reader.h" diff --git a/be/src/exec/runtime_filter/runtime_filter_producer.cpp b/be/src/exec/runtime_filter/runtime_filter_producer.cpp index 42d1938b790c2c..2a9839111eef41 100644 --- a/be/src/exec/runtime_filter/runtime_filter_producer.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_producer.cpp @@ -22,6 +22,7 @@ #include "exec/runtime_filter/runtime_filter_consumer.h" #include "exec/runtime_filter/runtime_filter_merger.h" #include "exec/runtime_filter/runtime_filter_wrapper.h" +#include "exec/runtime_filter/sync_size_callback.h" #include "util/brpc_client_cache.h" #include "util/brpc_closure.h" diff --git a/be/src/exec/runtime_filter/runtime_filter_producer.h b/be/src/exec/runtime_filter/runtime_filter_producer.h index 2fcac5bd82b979..1ecb8673a4727e 100644 --- a/be/src/exec/runtime_filter/runtime_filter_producer.h +++ b/be/src/exec/runtime_filter/runtime_filter_producer.h @@ -25,68 +25,10 @@ #include "exec/pipeline/dependency.h" #include "exec/runtime_filter/runtime_filter.h" #include "runtime/query_context.h" -#include "util/brpc_closure.h" namespace doris { -// Callback for sync-size RPCs. Handles errors (disable wrapper + sub dependency) in call(). -class SyncSizeCallback : public DummyBrpcCallback { - ENABLE_FACTORY_CREATOR(SyncSizeCallback); - -public: - SyncSizeCallback(std::shared_ptr dependency, - std::shared_ptr wrapper, - std::weak_ptr context) - : _dependency(std::move(dependency)), _wrapper(wrapper), _context(std::move(context)) {} - - void call() override { - // On error: disable the wrapper and sub the dependency here, because set_synced_size() - // will never be called (the merge node won't respond with a sync). - // On success: do NOT sub here. The merge node will respond with sync_filter_size, - // which calls set_synced_size() -> _dependency->sub(). - if (this->cntl_->Failed()) { - LOG(WARNING) << fmt::format("RPC meet failed: {}", this->cntl_->ErrorText()); - if (auto w = _wrapper.lock()) { - w->set_state(RuntimeFilterWrapper::State::DISABLED, this->cntl_->ErrorText()); - } - if (auto ctx = _context.lock()) { - if (!ctx->ignore_runtime_filter_error()) { - ctx->cancel( - Status::NetworkError("RPC meet failed: {}", this->cntl_->ErrorText())); - } - } - auto p = std::dynamic_pointer_cast(_dependency); - DORIS_CHECK(p); - p->sub(); - return; - } - - Status status = Status::create(this->response_->status()); - if (!status.ok()) { - LOG(WARNING) << "RPC meet error status: " << status; - if (auto w = _wrapper.lock()) { - w->set_state(RuntimeFilterWrapper::State::DISABLED, status.to_string()); - } - if (auto ctx = _context.lock()) { - if (!ctx->ignore_runtime_filter_error()) { - ctx->cancel(status); - } - } - auto p = std::dynamic_pointer_cast(_dependency); - DORIS_CHECK(p); - p->sub(); - } - } - -private: - std::shared_ptr _dependency; - // Should use weak ptr here, because when query context deconstructs, should also delete runtime filter - // context, it not the memory is not released. And rpc is in another thread, it will hold rf context - // after query context because the rpc is not returned. - std::weak_ptr _wrapper; - // input context is always meaningful. will check `ignore_runtime_filter_error` to decide whether to cancel query - std::weak_ptr _context; -}; +class SyncSizeCallback; // Work on (hash/corss) join build sink node, RuntimeFilterProducerHelper will manage all RuntimeFilterProducer // Used to generate specific predicate and publish it to consumer/merger diff --git a/be/src/exec/runtime_filter/sync_size_callback.h b/be/src/exec/runtime_filter/sync_size_callback.h new file mode 100644 index 00000000000000..42ab22db9510ff --- /dev/null +++ b/be/src/exec/runtime_filter/sync_size_callback.h @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "exec/pipeline/dependency.h" +#include "exec/runtime_filter/runtime_filter.h" +#include "runtime/query_context.h" +#include "util/brpc_closure.h" + +namespace doris { + +// Callback for sync-size RPCs. Handles errors (disable wrapper + sub dependency) in call(). +class SyncSizeCallback : public DummyBrpcCallback { + ENABLE_FACTORY_CREATOR(SyncSizeCallback); + +public: + SyncSizeCallback(std::shared_ptr dependency, + std::shared_ptr wrapper, + std::weak_ptr context) + : _dependency(std::move(dependency)), _wrapper(wrapper), _context(std::move(context)) {} + + void call() override { + // On error: disable the wrapper and sub the dependency here, because set_synced_size() + // will never be called (the merge node won't respond with a sync). + // On success: do NOT sub here. The merge node will respond with sync_filter_size, + // which calls set_synced_size() -> _dependency->sub(). + if (this->cntl_->Failed()) { + LOG(WARNING) << fmt::format("RPC meet failed: {}", this->cntl_->ErrorText()); + if (auto w = _wrapper.lock()) { + w->set_state(RuntimeFilterWrapper::State::DISABLED, this->cntl_->ErrorText()); + } + if (auto ctx = _context.lock()) { + if (!ctx->ignore_runtime_filter_error()) { + ctx->cancel( + Status::NetworkError("RPC meet failed: {}", this->cntl_->ErrorText())); + } + } + auto p = std::dynamic_pointer_cast(_dependency); + DORIS_CHECK(p); + p->sub(); + return; + } + + Status status = Status::create(this->response_->status()); + if (!status.ok()) { + LOG(WARNING) << "RPC meet error status: " << status; + if (auto w = _wrapper.lock()) { + w->set_state(RuntimeFilterWrapper::State::DISABLED, status.to_string()); + } + if (auto ctx = _context.lock()) { + if (!ctx->ignore_runtime_filter_error()) { + ctx->cancel(status); + } + } + auto p = std::dynamic_pointer_cast(_dependency); + DORIS_CHECK(p); + p->sub(); + } + } + +private: + std::shared_ptr _dependency; + // Should use weak ptr here, because when query context deconstructs, should also delete runtime filter + // context, it not the memory is not released. And rpc is in another thread, it will hold rf context + // after query context because the rpc is not returned. + std::weak_ptr _wrapper; + // input context is always meaningful. will check `ignore_runtime_filter_error` to decide whether to cancel query + std::weak_ptr _context; +}; +} // namespace doris diff --git a/be/src/exprs/function/function_date_or_datetime_to_string.cpp b/be/src/exprs/function/function_date_or_datetime_to_string.cpp index 88c9340622f812..1fc78a74cddb75 100644 --- a/be/src/exprs/function/function_date_or_datetime_to_string.cpp +++ b/be/src/exprs/function/function_date_or_datetime_to_string.cpp @@ -37,6 +37,7 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_date_or_datetime_v2.h" // IWYU pragma: keep +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "core/types.h" #include "exprs/aggregate/aggregate_function.h" diff --git a/be/src/exprs/function/function_datetime_floor_ceil.cpp b/be/src/exprs/function/function_datetime_floor_ceil.cpp index 6bf7e6c6799a92..1d4a0f1d739a56 100644 --- a/be/src/exprs/function/function_datetime_floor_ceil.cpp +++ b/be/src/exprs/function/function_datetime_floor_ceil.cpp @@ -46,6 +46,7 @@ #include "core/data_type/data_type_date_time.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "core/field.h" #include "core/pod_array_fwd.h" diff --git a/be/src/exprs/function/function_datetime_string_to_string.cpp b/be/src/exprs/function/function_datetime_string_to_string.cpp index 6ba9779c3cc12d..f1ba846e499b24 100644 --- a/be/src/exprs/function/function_datetime_string_to_string.cpp +++ b/be/src/exprs/function/function_datetime_string_to_string.cpp @@ -18,6 +18,7 @@ #include "exprs/function/function_datetime_string_to_string.h" #include "core/data_type/data_type_date_or_datetime_v2.h" // IWYU pragma: keep +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "exprs/function/date_time_transforms.h" #include "exprs/function/simple_function_factory.h" diff --git a/be/src/exprs/function/function_java_udf.cpp b/be/src/exprs/function/function_java_udf.cpp index a49f59d486f743..9ab09912988d59 100644 --- a/be/src/exprs/function/function_java_udf.cpp +++ b/be/src/exprs/function/function_java_udf.cpp @@ -24,6 +24,7 @@ #include #include "common/exception.h" +#include "common/metrics/doris_metrics.h" #include "core/block/block.h" #include "format/jni/jni_data_bridge.h" #include "jni.h" diff --git a/be/src/exprs/function/function_other_types_to_date.cpp b/be/src/exprs/function/function_other_types_to_date.cpp index cdcdaea146b8f8..993ea4a9539bec 100644 --- a/be/src/exprs/function/function_other_types_to_date.cpp +++ b/be/src/exprs/function/function_other_types_to_date.cpp @@ -51,6 +51,7 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" #include "core/pod_array.h" diff --git a/be/src/exprs/function/function_variant_type.cpp b/be/src/exprs/function/function_variant_type.cpp index f7e21bc9a6a769..9b4d49d3b07c81 100644 --- a/be/src/exprs/function/function_variant_type.cpp +++ b/be/src/exprs/function/function_variant_type.cpp @@ -24,6 +24,7 @@ #include "core/column/column_string.h" #include "core/column/column_vector.h" #include "core/column/variant_v2/column_variant_v2.h" +#include "core/data_type/data_type_string.h" #include "exprs/function/simple_function_factory.h" #include "util/string_util.h" diff --git a/be/src/exprs/function/time_of_function.cpp b/be/src/exprs/function/time_of_function.cpp index 5a1bf52cfca14d..bde29efc9169c8 100644 --- a/be/src/exprs/function/time_of_function.cpp +++ b/be/src/exprs/function/time_of_function.cpp @@ -20,6 +20,7 @@ #include "core/data_type/data_type_date_or_datetime_v2.h" // IWYU pragma: keep #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "exprs/function/date_time_transforms.h" #include "exprs/function/function_date_or_datetime_to_something.h" #include "exprs/function/simple_function_factory.h" diff --git a/be/src/exprs/function/to_time_function.cpp b/be/src/exprs/function/to_time_function.cpp index 557988dc06951c..351d2054d3a9aa 100644 --- a/be/src/exprs/function/to_time_function.cpp +++ b/be/src/exprs/function/to_time_function.cpp @@ -19,6 +19,7 @@ #include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_date_time.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/types.h" #include "exprs/function/date_time_transforms.h" #include "exprs/function/function_date_or_datetime_to_something.h" diff --git a/be/src/exprs/vexpr_context.h b/be/src/exprs/vexpr_context.h index 9e0245a4bc43c1..367aeef515d81b 100644 --- a/be/src/exprs/vexpr_context.h +++ b/be/src/exprs/vexpr_context.h @@ -42,7 +42,7 @@ #include "storage/index/ann/ann_search_params.h" #include "storage/index/inverted/inverted_index_reader.h" #include "storage/index/zone_map/zonemap_filter_result.h" -#include "storage/segment/column_reader.h" +#include "storage/segment/column_iterator_options.h" namespace doris { class RowDescriptor; diff --git a/be/src/format_v2/materialized_reader_util.cpp b/be/src/format_v2/materialized_reader_util.cpp index ca2c4c86b43a93..8bfe66479c1532 100644 --- a/be/src/format_v2/materialized_reader_util.cpp +++ b/be/src/format_v2/materialized_reader_util.cpp @@ -26,6 +26,7 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_struct.h" #include "core/types.h" +#include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "format_v2/file_reader.h" #include "io/io_common.h" diff --git a/be/src/format_v2/table/fluss_union_lake_reader.cpp b/be/src/format_v2/table/fluss_union_lake_reader.cpp index b381ac676840cc..7de121cc7cd42b 100644 --- a/be/src/format_v2/table/fluss_union_lake_reader.cpp +++ b/be/src/format_v2/table/fluss_union_lake_reader.cpp @@ -30,6 +30,7 @@ #include "core/column/column_vector.h" #include "exprs/vexpr_context.h" #include "exprs/vslot_ref.h" +#include "format/format_common.h" #include "format_v2/column_mapper.h" #include "format_v2/expr/equality_delete_predicate.h" #include "format_v2/jni/fluss_jni_reader.h" diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 82d086d417e3ae..210e06c560f55d 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -49,6 +49,7 @@ #include "core/value/time_value.h" #include "exprs/vliteral.h" #include "exprs/vslot_ref.h" +#include "format/format_common.h" #include "format/table/deletion_vector_reader.h" #include "format/table/iceberg_default_value.h" #include "format_v2/expr/equality_delete_predicate.h" diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 154ba5e64310d4..10bdcd21357f39 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -71,6 +71,7 @@ namespace doris { class Block; struct DeleteFileDesc; class RuntimeState; +class ShardedKVCache; } // namespace doris namespace doris::format { diff --git a/be/src/information_schema/schema_column_data_sizes_scanner.h b/be/src/information_schema/schema_column_data_sizes_scanner.h index 03e2e80779c5aa..29e1e8c8d10bc4 100644 --- a/be/src/information_schema/schema_column_data_sizes_scanner.h +++ b/be/src/information_schema/schema_column_data_sizes_scanner.h @@ -23,6 +23,7 @@ #include #include "information_schema/schema_scanner.h" +#include "storage/rowset/rowset_fwd.h" namespace doris { class RuntimeState; @@ -69,4 +70,4 @@ class SchemaColumnDataSizesScanner : public SchemaScanner { size_t _column_data_sizes_idx; }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/io/cache/async_cache_write_manager.cpp b/be/src/io/cache/async_cache_write_manager.cpp index fcf07f5f45692e..22ec0a83e4c440 100644 --- a/be/src/io/cache/async_cache_write_manager.cpp +++ b/be/src/io/cache/async_cache_write_manager.cpp @@ -38,6 +38,7 @@ #include "runtime/thread_context.h" #include "util/countdown_latch.h" #include "util/defer_op.h" +#include "util/threadpool.h" #include "util/time.h" namespace doris::io { diff --git a/be/src/io/cache/async_cache_write_manager.h b/be/src/io/cache/async_cache_write_manager.h index 2980b8d74a5d0e..319ab74ddedd7b 100644 --- a/be/src/io/cache/async_cache_write_manager.h +++ b/be/src/io/cache/async_cache_write_manager.h @@ -32,7 +32,10 @@ #include "io/cache/file_cache_common.h" #include "runtime/memory/mem_tracker_limiter.h" #include "util/slice.h" -#include "util/threadpool.h" + +namespace doris { +class ThreadPool; +} namespace doris::io { diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index 13d1fff0fba9ac..2def50f605a660 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -43,7 +43,6 @@ #include "io/cache/file_cache_storage.h" #include "io/cache/lru_queue_recorder.h" #include "runtime/runtime_profile.h" -#include "util/threadpool.h" namespace doris::io { using RecycleFileCacheKeys = moodycamel::ConcurrentQueue; diff --git a/be/src/io/cache/block_file_cache_factory.cpp b/be/src/io/cache/block_file_cache_factory.cpp index 69a64bd39218a5..5e1559af94f777 100644 --- a/be/src/io/cache/block_file_cache_factory.cpp +++ b/be/src/io/cache/block_file_cache_factory.cpp @@ -20,6 +20,7 @@ #include "io/cache/block_file_cache_factory.h" +#include #include #include @@ -48,6 +49,7 @@ #include "runtime/exec_env.h" #include "runtime/thread_context.h" #include "service/backend_options.h" +#include "storage/options.h" #include "util/mem_info.h" #include "util/slice.h" diff --git a/be/src/io/cache/block_file_cache_factory.h b/be/src/io/cache/block_file_cache_factory.h index 88de27883e115a..dec086e986c92f 100644 --- a/be/src/io/cache/block_file_cache_factory.h +++ b/be/src/io/cache/block_file_cache_factory.h @@ -20,8 +20,6 @@ #pragma once -#include - #include #include #include @@ -33,9 +31,10 @@ #include "common/status.h" #include "io/cache/block_file_cache.h" #include "io/cache/file_cache_common.h" -#include "storage/options.h" namespace doris { +class CacheBlockPB; class TUniqueId; +struct CachePath; class Block; diff --git a/be/src/io/cache/file_cache_common.cpp b/be/src/io/cache/file_cache_common.cpp index 948bd4e144ecde..636fdb220687b8 100644 --- a/be/src/io/cache/file_cache_common.cpp +++ b/be/src/io/cache/file_cache_common.cpp @@ -23,6 +23,7 @@ #include "common/config.h" #include "exec/common/hex.h" #include "io/cache/block_file_cache.h" +#include "storage/olap_define.h" namespace doris::io { diff --git a/be/src/io/fs/hdfs_file_writer.cpp b/be/src/io/fs/hdfs_file_writer.cpp index aef58205b6d7b1..94f623956c4447 100644 --- a/be/src/io/fs/hdfs_file_writer.cpp +++ b/be/src/io/fs/hdfs_file_writer.cpp @@ -45,6 +45,7 @@ #include "service/backend_options.h" #include "util/bvar_helper.h" #include "util/jni-util.h" +#include "util/threadpool.h" namespace doris::io { diff --git a/be/src/io/fs/packed_file_manager.h b/be/src/io/fs/packed_file_manager.h index de9ebf89df7462..6a990429e37f70 100644 --- a/be/src/io/fs/packed_file_manager.h +++ b/be/src/io/fs/packed_file_manager.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include #include @@ -40,6 +39,10 @@ #include "io/fs/path.h" #include "util/slice.h" +namespace doris::cloud { +class PackedFileInfoPB; +} + namespace doris::io { // Upload state of the packed file a slice belongs to diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index ec18463b559815..b5529470974fb1 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -38,6 +38,7 @@ #include "io/cache/block_file_cache.h" #include "io/fs/err_utils.h" #include "io/fs/read_io_trace.h" +#include "io/fs/s3_file_system.h" #include "io/io_common.h" #include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" diff --git a/be/src/io/fs/s3_file_reader.h b/be/src/io/fs/s3_file_reader.h index 40e4a2d23042fd..324c7621713d34 100644 --- a/be/src/io/fs/s3_file_reader.h +++ b/be/src/io/fs/s3_file_reader.h @@ -25,13 +25,13 @@ #include "io/fs/file_reader.h" #include "io/fs/file_system.h" #include "io/fs/path.h" -#include "io/fs/s3_file_system.h" #include "util/slice.h" namespace doris { class RuntimeProfile; namespace io { +class ObjClientHolder; struct IOContext; class S3FileReader final : public FileReader { diff --git a/be/src/io/fs/stream_sink_file_writer.h b/be/src/io/fs/stream_sink_file_writer.h index 74f7d4e091ca32..d65c3c22a268f2 100644 --- a/be/src/io/fs/stream_sink_file_writer.h +++ b/be/src/io/fs/stream_sink_file_writer.h @@ -19,6 +19,7 @@ #include #include +#include #include #include diff --git a/be/src/io/hdfs_util.h b/be/src/io/hdfs_util.h index d7a74134db64ec..b1fc82759e6940 100644 --- a/be/src/io/hdfs_util.h +++ b/be/src/io/hdfs_util.h @@ -30,11 +30,11 @@ #include "io/fs/hdfs.h" #include "io/fs/path.h" +namespace doris { namespace cloud { class HdfsVaultInfo; } -namespace doris { class HDFSCommonBuilder; class THdfsParams; diff --git a/be/src/runtime/query_context.h b/be/src/runtime/query_context.h index 8dcd78fd2356bd..b2b0184589a52f 100644 --- a/be/src/runtime/query_context.h +++ b/be/src/runtime/query_context.h @@ -37,6 +37,7 @@ #include "exec/common/memory.h" #include "exec/runtime_filter/runtime_filter_mgr.h" #include "exec/scan/scanner_scheduler.h" +#include "io/cache/block_file_cache.h" #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/runtime_predicate.h" diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 1a69a1ce4cbec3..1822fd836ca4eb 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -58,6 +59,19 @@ namespace doris { +segment_v2::CompressionTypePB RuntimeState::fragement_transmission_compression_type() const { + if (_query_options.__isset.fragment_transmission_compression_codec) { + if (_query_options.fragment_transmission_compression_codec == "lz4") { + return segment_v2::CompressionTypePB::LZ4; + } else if (_query_options.fragment_transmission_compression_codec == "snappy") { + return segment_v2::CompressionTypePB::SNAPPY; + } else { + return segment_v2::CompressionTypePB::NO_COMPRESSION; + } + } + return segment_v2::CompressionTypePB::NO_COMPRESSION; +} + ExternalFileReportReservation::~ExternalFileReportReservation() { if (_state != nullptr) { _state->release_external_file_report_reservation(_reserved_bytes); diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 8bc987949d45d1..02c47c235edab1 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -22,7 +22,6 @@ #include #include -#include #include #include @@ -50,6 +49,9 @@ #include "util/timezone_utils.h" namespace doris { +namespace segment_v2 { +enum CompressionTypePB : int; +} namespace io { class S3FileSystem; } // namespace io @@ -473,18 +475,7 @@ class RuntimeState { _query_options.enable_fuzzy_blockable_task; } - segment_v2::CompressionTypePB fragement_transmission_compression_type() const { - if (_query_options.__isset.fragment_transmission_compression_codec) { - if (_query_options.fragment_transmission_compression_codec == "lz4") { - return segment_v2::CompressionTypePB::LZ4; - } else if (_query_options.fragment_transmission_compression_codec == "snappy") { - return segment_v2::CompressionTypePB::SNAPPY; - } else { - return segment_v2::CompressionTypePB::NO_COMPRESSION; - } - } - return segment_v2::CompressionTypePB::NO_COMPRESSION; - } + segment_v2::CompressionTypePB fragement_transmission_compression_type() const; bool skip_storage_engine_merge() const { return _query_options.__isset.skip_storage_engine_merge && diff --git a/be/src/runtime/workload_group/workload_group_manager.h b/be/src/runtime/workload_group/workload_group_manager.h index ea3795942601df..5821d569633030 100644 --- a/be/src/runtime/workload_group/workload_group_manager.h +++ b/be/src/runtime/workload_group/workload_group_manager.h @@ -19,12 +19,17 @@ #include #include +#include +#include #include #include +#include #include +#include #include "common/be_mock_util.h" -#include "runtime/workload_group/workload_group.h" +#include "common/status.h" +#include "runtime/workload_group/workload_group_fwd.h" #include "util/time.h" namespace doris { @@ -35,6 +40,8 @@ class Block; class TaskScheduler; class MultiCoreTaskQueue; +class ResourceContext; +struct WorkloadGroupInfo; // In doris, query includes all tasks such as query, load, compaction, schemachange, etc. class PausedQuery { diff --git a/be/src/service/http/http_channel.cpp b/be/src/service/http/http_channel.cpp index bf101b83f4d8de..359ff487939200 100644 --- a/be/src/service/http/http_channel.cpp +++ b/be/src/service/http/http_channel.cpp @@ -22,6 +22,9 @@ #include #include #include +#include +#include +#include #include #include diff --git a/be/src/storage/id_manager.h b/be/src/storage/id_manager.h index 418483b79dda95..98703b80773287 100644 --- a/be/src/storage/id_manager.h +++ b/be/src/storage/id_manager.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include #include #include @@ -289,4 +288,4 @@ class IdManager { std::shared_mutex _query_to_id_file_map_mtx; }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/storage/segment/column_iterator_options.h b/be/src/storage/segment/column_iterator_options.h new file mode 100644 index 00000000000000..42daf6a32a4d7f --- /dev/null +++ b/be/src/storage/segment/column_iterator_options.h @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "io/io_common.h" + +namespace doris { +struct OlapReaderStatistics; +namespace io { +class FileReader; +} +namespace segment_v2 { + +struct ColumnIteratorOptions { + bool use_page_cache = false; + bool is_predicate_column = false; + // for page cache allocation + // page types are divided into DATA_PAGE & INDEX_PAGE + // INDEX_PAGE including index_page, dict_page and short_key_page + PageTypePB type = PageTypePB::UNKNOWN_PAGE_TYPE; + io::FileReader* file_reader = nullptr; // Ref + // reader statistics + OlapReaderStatistics* stats = nullptr; // Ref + io::IOContext io_ctx; + bool only_read_offsets = false; + + void sanity_check() const { + CHECK_NOTNULL(file_reader); + CHECK_NOTNULL(stats); + } +}; + +} // namespace segment_v2 +} // namespace doris diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h index d03532025c8119..171e746cff6a7f 100644 --- a/be/src/storage/segment/column_reader.h +++ b/be/src/storage/segment/column_reader.h @@ -47,6 +47,7 @@ #include "storage/index/zone_map/zone_map_index.h" #include "storage/olap_common.h" #include "storage/predicate/column_predicate.h" +#include "storage/segment/column_iterator_options.h" #include "storage/segment/common.h" #include "storage/segment/page_handle.h" // for PageHandle #include "storage/segment/page_pointer.h" @@ -104,25 +105,6 @@ struct ColumnReaderOptions { TabletSchemaSPtr tablet_schema = nullptr; }; -struct ColumnIteratorOptions { - bool use_page_cache = false; - bool is_predicate_column = false; - // for page cache allocation - // page types are divided into DATA_PAGE & INDEX_PAGE - // INDEX_PAGE including index_page, dict_page and short_key_page - PageTypePB type = PageTypePB::UNKNOWN_PAGE_TYPE; - io::FileReader* file_reader = nullptr; // Ref - // reader statistics - OlapReaderStatistics* stats = nullptr; // Ref - io::IOContext io_ctx; - bool only_read_offsets = false; - - void sanity_check() const { - CHECK_NOTNULL(file_reader); - CHECK_NOTNULL(stats); - } -}; - class ColumnIterator; class OffsetFileColumnIterator; class FileColumnIterator; diff --git a/be/src/storage/tablet/tablet_manager.cpp b/be/src/storage/tablet/tablet_manager.cpp index 54c8e4a68e8eb9..7cdbbd2607caa2 100644 --- a/be/src/storage/tablet/tablet_manager.cpp +++ b/be/src/storage/tablet/tablet_manager.cpp @@ -86,7 +86,8 @@ bvar::Adder g_tablet_meta_schema_columns_count("tablet_meta_schema_colu TabletManager::TabletManager(StorageEngine& engine, int32_t tablet_map_lock_shard_size) : _engine(engine), _tablets_shards_size(tablet_map_lock_shard_size), - _tablets_shards_mask(tablet_map_lock_shard_size - 1) { + _tablets_shards_mask(tablet_map_lock_shard_size - 1), + _tablet_stat_list_cache(std::make_shared>()) { CHECK_GT(_tablets_shards_size, 0); CHECK_EQ(_tablets_shards_size & _tablets_shards_mask, 0); _tablets_shards.resize(_tablets_shards_size); diff --git a/be/src/storage/tablet/tablet_manager.h b/be/src/storage/tablet/tablet_manager.h index 7c0e02670f36c0..2ae6269db6bd5a 100644 --- a/be/src/storage/tablet/tablet_manager.h +++ b/be/src/storage/tablet/tablet_manager.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include #include #include @@ -50,6 +49,9 @@ class MemTracker; class TCreateTabletReq; class TTablet; class TTabletInfo; +class TTabletStat; +class TTabletStatResult; +class TCheckStorageFormatResult; // TabletManager provides get, add, delete tablet method for storage engine // NOTE: If you want to add a method that needs to hold meta-lock before you can call it, @@ -274,8 +276,7 @@ class TabletManager { std::mutex _gc_tablets_lock; std::mutex _tablet_stat_cache_mutex; - std::shared_ptr> _tablet_stat_list_cache = - std::make_shared>(); + std::shared_ptr> _tablet_stat_list_cache; tablet_map_t& _get_tablet_map(TTabletId tablet_id); diff --git a/be/src/storage/tablet/tablet_meta.h b/be/src/storage/tablet/tablet_meta.h index 5001db7af31c3c..2c7b1720c78ce8 100644 --- a/be/src/storage/tablet/tablet_meta.h +++ b/be/src/storage/tablet/tablet_meta.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include #include diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index f3487fd9cc98d0..b275c1c2889ac7 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include "util/string_util.h" diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 6856d205ff5241..47ec7293341d3c 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -23,7 +23,6 @@ #include #include #include -#include #ifdef BE_TEST #include #endif @@ -53,6 +52,10 @@ class Adder; namespace doris { +namespace cloud { +class ObjectStoreInfoPB; +} + std::string hide_access_key(const std::string& ak); class S3URI; diff --git a/be/src/util/time.cpp b/be/src/util/time.cpp index 7a7f510905687d..792b46e5b3f57d 100644 --- a/be/src/util/time.cpp +++ b/be/src/util/time.cpp @@ -67,13 +67,13 @@ static std::string FormatSubSecond(const system_clock::time_point& t, TimePrecis auto frac = t.time_since_epoch(); if (p == TimePrecision::Millisecond) { auto subsec = duration_cast(frac) % MILLIS_PER_SEC; - ss << "." << std::setfill('0') << std::setw(3) << abs(subsec.count()); + ss << "." << std::setfill('0') << std::setw(3) << std::abs(subsec.count()); } else if (p == TimePrecision::Microsecond) { auto subsec = duration_cast(frac) % MICROS_PER_SEC; - ss << "." << std::setfill('0') << std::setw(6) << abs(subsec.count()); + ss << "." << std::setfill('0') << std::setw(6) << std::abs(subsec.count()); } else if (p == TimePrecision::Nanosecond) { auto subsec = duration_cast(frac) % NANOS_PER_SEC; - ss << "." << std::setfill('0') << std::setw(9) << abs(subsec.count()); + ss << "." << std::setfill('0') << std::setw(9) << std::abs(subsec.count()); } else { // 1-second precision or unknown unit. Return empty string. DCHECK_EQ(TimePrecision::Second, p); diff --git a/be/src/util/uid_util.cpp b/be/src/util/uid_util.cpp index 4021bf68ac1bd8..0e1ecbe8afe9f7 100644 --- a/be/src/util/uid_util.cpp +++ b/be/src/util/uid_util.cpp @@ -28,6 +28,50 @@ #include "util/hash_util.hpp" namespace doris { +UniqueId::UniqueId(const TUniqueId& tuid) : hi(tuid.hi), lo(tuid.lo) {} +UniqueId::UniqueId(const PUniqueId& puid) : hi(puid.hi()), lo(puid.lo()) {} + +UniqueId& UniqueId::operator=(const PUniqueId puid) { + hi = puid.hi(); + lo = puid.lo(); + return *this; +} + +UniqueId& UniqueId::operator=(const TUniqueId tuid) { + hi = tuid.hi; + lo = tuid.lo; + return *this; +} + +bool UniqueId::operator==(const PUniqueId& rhs) const { + return hi == rhs.hi() && lo == rhs.lo(); +} +bool UniqueId::operator!=(const PUniqueId& rhs) const { + return hi != rhs.hi() || lo != rhs.lo(); +} + +TUniqueId UniqueId::to_thrift() const { + TUniqueId tid; + tid.__set_hi(hi); + tid.__set_lo(lo); + return tid; +} + +PUniqueId UniqueId::to_proto() const { + PUniqueId pid; + pid.set_hi(hi); + pid.set_lo(lo); + return pid; +} + +TUniqueId generate_uuid() { + auto uuid = boost::uuids::basic_random_generator()(); + TUniqueId uid; + memcpy(&uid.hi, uuid.data, sizeof(int64_t)); + memcpy(&uid.lo, uuid.data + sizeof(int64_t), sizeof(int64_t)); + return uid; +} + size_t UniqueId::hash(size_t seed) const { return doris::HashUtil::hash(this, sizeof(*this), cast_set(seed)); } diff --git a/be/src/util/uid_util.h b/be/src/util/uid_util.h index a459ea92a16ceb..e2cb6ebf642442 100644 --- a/be/src/util/uid_util.h +++ b/be/src/util/uid_util.h @@ -17,8 +17,6 @@ #pragma once -#include -#include #include #include @@ -34,6 +32,9 @@ namespace doris { +class TUniqueId; +class PUniqueId; + // convert int to a hex format string, buf must enough to hold converted hex string template void to_hex(T val, char* buf) { @@ -67,8 +68,8 @@ struct UniqueId { UniqueId() = default; UniqueId(int64_t hi_, int64_t lo_) : hi(hi_), lo(lo_) {} UniqueId(const UniqueId& uid) : hi(uid.hi), lo(uid.lo) {} - UniqueId(const TUniqueId& tuid) : hi(tuid.hi), lo(tuid.lo) {} - UniqueId(const PUniqueId& puid) : hi(puid.hi()), lo(puid.lo()) {} + UniqueId(const TUniqueId& tuid); + UniqueId(const PUniqueId& puid); UniqueId(const std::string& hi_str, const std::string& lo_str) { from_hex(&hi, hi_str); from_hex(&lo, lo_str); @@ -101,21 +102,13 @@ struct UniqueId { return *this; } - UniqueId& operator=(const PUniqueId puid) { - hi = puid.hi(); - lo = puid.lo(); - return *this; - } + UniqueId& operator=(const PUniqueId puid); - UniqueId& operator=(const TUniqueId tuid) { - hi = tuid.hi; - lo = tuid.lo; - return *this; - } + UniqueId& operator=(const TUniqueId tuid); //compare PUniqueId and UniqueId - bool operator==(const PUniqueId& rhs) const { return hi == rhs.hi() && lo == rhs.lo(); } + bool operator==(const PUniqueId& rhs) const; - bool operator!=(const PUniqueId& rhs) const { return hi != rhs.hi() || lo != rhs.lo(); } + bool operator!=(const PUniqueId& rhs) const; // std::map std::set needs this operator bool operator<(const UniqueId& right) const { @@ -134,19 +127,9 @@ struct UniqueId { bool operator!=(const UniqueId& rhs) const { return hi != rhs.hi || lo != rhs.lo; } - TUniqueId to_thrift() const { - TUniqueId tid; - tid.__set_hi(hi); - tid.__set_lo(lo); - return tid; - } + TUniqueId to_thrift() const; - PUniqueId to_proto() const { - PUniqueId pid; - pid.set_hi(hi); - pid.set_lo(lo); - return pid; - } + PUniqueId to_proto() const; }; // This function must be called 'hash_value' to be picked up by boost. @@ -158,13 +141,7 @@ inline std::string generate_uuid_string() { } /// generates a 16 byte UUID -inline TUniqueId generate_uuid() { - auto uuid = boost::uuids::basic_random_generator()(); - TUniqueId uid; - memcpy(&uid.hi, uuid.data, sizeof(int64_t)); - memcpy(&uid.lo, uuid.data + sizeof(int64_t), sizeof(int64_t)); - return uid; -} +TUniqueId generate_uuid(); std::ostream& operator<<(std::ostream& os, const UniqueId& uid); diff --git a/be/test/cloud/cloud_schema_change_job_test.cpp b/be/test/cloud/cloud_schema_change_job_test.cpp index 2b7e7808367f8e..5915e3cc6ed724 100644 --- a/be/test/cloud/cloud_schema_change_job_test.cpp +++ b/be/test/cloud/cloud_schema_change_job_test.cpp @@ -18,6 +18,7 @@ #include "cloud/cloud_schema_change_job.h" #include +#include #include #include diff --git a/be/test/exec/pipeline/pipeline_task_test.cpp b/be/test/exec/pipeline/pipeline_task_test.cpp index 12bc33ad4e3afa..47496236913c6b 100644 --- a/be/test/exec/pipeline/pipeline_task_test.cpp +++ b/be/test/exec/pipeline/pipeline_task_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include @@ -39,6 +40,7 @@ #include "load/stream_load/new_load_stream_mgr.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" +#include "runtime/workload_group/workload_group.h" #include "testutil/mock/mock_runtime_state.h" #include "testutil/mock/mock_thread_mem_tracker_mgr.h" #include "testutil/mock/mock_workload_group_mgr.h" diff --git a/be/test/exec/runtime_filter/sync_size_callback_test.cpp b/be/test/exec/runtime_filter/sync_size_callback_test.cpp index a088202090f5e2..c5fe75f4b0de30 100644 --- a/be/test/exec/runtime_filter/sync_size_callback_test.cpp +++ b/be/test/exec/runtime_filter/sync_size_callback_test.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include "exec/runtime_filter/sync_size_callback.h" + #include #include diff --git a/be/test/exprs/index_exec_context_test.cpp b/be/test/exprs/index_exec_context_test.cpp index 2b408b61caf75d..871b4f229b6d66 100644 --- a/be/test/exprs/index_exec_context_test.cpp +++ b/be/test/exprs/index_exec_context_test.cpp @@ -24,6 +24,7 @@ #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" +#include "storage/index/index_iterator.h" #include "storage/index/inverted/inverted_index_reader.h" #include "storage/olap_common.h" #include "storage/schema.h" diff --git a/be/test/exprs/vexpr_evalute_inverted_index_test.cpp b/be/test/exprs/vexpr_evalute_inverted_index_test.cpp index 4bb51523d26400..aee736445d6732 100644 --- a/be/test/exprs/vexpr_evalute_inverted_index_test.cpp +++ b/be/test/exprs/vexpr_evalute_inverted_index_test.cpp @@ -29,6 +29,7 @@ #include "gtest/gtest.h" #include "runtime/descriptors.h" #include "runtime/runtime_state.h" +#include "storage/index/index_iterator.h" #include "storage/schema.h" // test expr is: v not in (ipv4_literal, casttoipv4(varchar)) diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 1dc88ace240d90..52a6182590e995 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -54,6 +54,7 @@ #include "format_v2/parquet/reader/native/page_reader.h" #include "io/fs/buffered_reader.h" #include "io/fs/file_reader.h" +#include "storage/segment/row_ranges.h" #include "util/block_compression.h" #include "util/coding.h" #include "util/faststring.h" @@ -1551,7 +1552,7 @@ TEST(ParquetV2NativeDecoderTest, DictionaryProbeMaterializesTypedValuesOnlyOnce) field.parquet_schema.__set_type(tparquet::Type::INT32); field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); auto file = std::make_shared(bytes); - const auto row_ranges = ::doris::RowRanges::create_single(2); + const auto row_ranges = segment_v2::RowRanges::create_single(2); ScalarColumnReader reader(row_ranges, 2, chunk, nullptr, nullptr, nullptr); ASSERT_TRUE(reader.init(file, &field, bytes.size(), nullptr, "", ParquetReaderCompat {}, true) .ok()); @@ -3418,7 +3419,7 @@ TEST(ParquetV2NativeDecoderTest, LazyFlatIndexedSkipValidatesFirstPageCardinalit field.parquet_schema.__set_type(tparquet::Type::INT32); field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); auto file = std::make_shared(bytes); - const auto row_ranges = ::doris::RowRanges::create_single(2, 4); + const auto row_ranges = segment_v2::RowRanges::create_single(2, 4); ScalarColumnReader reader(row_ranges, 6, chunk, &offset_index, nullptr, nullptr); ASSERT_TRUE(reader.init(file, &field, bytes.size(), nullptr, "", ParquetReaderCompat {}, true) .ok()); @@ -3464,7 +3465,7 @@ TEST(ParquetV2NativeDecoderTest, LazyFixedWidthFilterUsesReconciledFirstPageRang field.parquet_schema.__set_type(tparquet::Type::INT32); field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); auto file = std::make_shared(bytes); - const auto row_ranges = ::doris::RowRanges::create_single(0, 2); + const auto row_ranges = segment_v2::RowRanges::create_single(0, 2); ScalarColumnReader reader(row_ranges, 3, chunk, &offset_index, nullptr, nullptr); ASSERT_TRUE(reader.init(file, &field, bytes.size(), nullptr, "", ParquetReaderCompat {}, true) .ok()); @@ -3515,7 +3516,7 @@ TEST(ParquetV2NativeDecoderTest, LazyFlatIndexedFallbackUsesReconciledFirstPageR field.parquet_schema.__set_type(tparquet::Type::INT32); field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); auto file = std::make_shared(bytes); - const auto row_ranges = ::doris::RowRanges::create_single(0, 2); + const auto row_ranges = segment_v2::RowRanges::create_single(0, 2); ScalarColumnReader reader(row_ranges, 3, chunk, &offset_index, nullptr, nullptr); ASSERT_TRUE(reader.init(file, &field, bytes.size(), nullptr, "", ParquetReaderCompat {}, true) .ok()); @@ -3792,7 +3793,7 @@ TEST(ParquetV2NativeDecoderTest, LevelOnlyReaderSkipsLeadingZeroValuePages) { field.parquet_schema.__set_type(tparquet::Type::INT32); field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); - auto row_ranges = ::doris::RowRanges::create_single(1); + auto row_ranges = segment_v2::RowRanges::create_single(1); ScalarColumnReader scalar_reader(row_ranges, 1, chunk, nullptr, &utc, nullptr); ASSERT_TRUE( @@ -4878,7 +4879,7 @@ TEST(ParquetV2NativeDecoderTest, CacheDisabledPagesDoNotPrepareCopiedPayload) { } TEST(ParquetV2NativeDecoderTest, OversizedNestedBatchScratchUsesIdleBatchHysteresis) { - ::doris::RowRanges row_ranges; + segment_v2::RowRanges row_ranges; tparquet::ColumnChunk chunk; ScalarColumnReader reader(row_ranges, 1, chunk, nullptr, nullptr, nullptr); @@ -4898,7 +4899,7 @@ TEST(ParquetV2NativeDecoderTest, OversizedNestedBatchScratchUsesIdleBatchHystere } TEST(ParquetV2NativeDecoderTest, NestedBatchScratchEnforcesAggregateRetentionLimit) { - ::doris::RowRanges row_ranges; + segment_v2::RowRanges row_ranges; tparquet::ColumnChunk chunk; ScalarColumnReader reader(row_ranges, 1, chunk, nullptr, nullptr, nullptr); diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index d7c4c2cd5a6129..aaa684be5138a9 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -37,6 +37,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/field.h" +#include "format/format_common.h" #include "format_v2/column_data.h" #include "format_v2/parquet/parquet_reader.h" #include "gen_cpp/ExternalTableSchema_types.h" diff --git a/be/test/io/client/s3_file_system_test.cpp b/be/test/io/client/s3_file_system_test.cpp index b7ba5966843ea3..0e03d130e988b0 100644 --- a/be/test/io/client/s3_file_system_test.cpp +++ b/be/test/io/client/s3_file_system_test.cpp @@ -38,6 +38,7 @@ #include "util/defer_op.h" #include "util/s3_rate_limiter_manager.h" #include "util/s3_util.h" +#include "util/threadpool.h" namespace doris { diff --git a/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp b/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp index 0372474aa03361..ce13fbb345920c 100644 --- a/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp +++ b/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp @@ -26,6 +26,7 @@ #include "runtime/fragment_mgr.h" #include "runtime/frontend_info.h" #include "runtime/index_policy/index_policy_mgr.h" +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_group/workload_group_manager.h" #include "storage/id_manager.h" #include "util/debug_points.h" diff --git a/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp b/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp index 188f16e96a8ee6..6ae8b6fad54770 100644 --- a/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp +++ b/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp @@ -23,6 +23,7 @@ #include "gtest/gtest_pred_impl.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_group/workload_group_manager.h" #include "runtime/workload_management/resource_context.h" diff --git a/be/test/storage/compaction/cloud_index_change_task_test.cpp b/be/test/storage/compaction/cloud_index_change_task_test.cpp index 33ad9a338207ed..043f4ae76cdaf3 100644 --- a/be/test/storage/compaction/cloud_index_change_task_test.cpp +++ b/be/test/storage/compaction/cloud_index_change_task_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include "cloud/cloud_storage_engine.h" @@ -333,4 +334,4 @@ TEST_F(CloudIndexChangeTaskTest, register_task_conflict_test) { ASSERT_FALSE(ret); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/util/uid_util_test.cpp b/be/test/util/uid_util_test.cpp index bc105a185fc852..f56b6e865bc899 100644 --- a/be/test/util/uid_util_test.cpp +++ b/be/test/util/uid_util_test.cpp @@ -17,6 +17,8 @@ #include "util/uid_util.h" +#include +#include #include #include From dc2a6a3dd587865ae137d5cc5a9e85bb2359b7b0 Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Mon, 2 Nov 2026 19:23:10 +0800 Subject: [PATCH 2/3] [improvement](build) Update header dependency budgets after include cleanup ### What problem does this PR solve? Related PR: #69717 Problem Summary: The include cleanup reduced transitive header dependencies, leaving several budgets above the measured reach. Update the forward closure budget and reverse reach baselines to preserve these reductions, with room for subsequent changes. Retain the existing 10% reverse reach allowance and gate behavior. Set the FrontendService_types.h, data.pb.h, storage/options.h and PaloInternalService_types.h baselines to 350, 120, 350 and 1150 respectively. ### Release note None ### Check List (For Author) - Test: build-support/check-build-hygiene.sh; build-support/tests/run.sh; git diff --check - Behavior changed: Yes, header dependency budget values are updated. - Does this need documentation: No --- build-support/check-header-deps.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/build-support/check-header-deps.py b/build-support/check-header-deps.py index e4e82135c0a30e..1de985c12b0509 100755 --- a/build-support/check-header-deps.py +++ b/build-support/check-header-deps.py @@ -551,7 +551,7 @@ "core/column/column.h": 66, "exprs/function/function.h": 106, # This dependency hub includes the shared read SPI and SNII gram metadata. - "exec/pipeline/dependency.h": 359, + "exec/pipeline/dependency.h": 248, "pch/pch.h": 8, } @@ -567,23 +567,23 @@ "gen_cpp/Status_types.h": 37, # Still carried legitimately by util/uid_util.h (PUniqueId by value) and # the profile family; the status.h cut brought these down from 2458/2091. - "gen_cpp/types.pb.h": 1791, + "gen_cpp/types.pb.h": 1250, "gen_cpp/Types_types.h": 1993, - "gen_cpp/segment_v2.pb.h": 1234, - "gen_cpp/PaloInternalService_types.h": 1133, - "util/threadpool.h": 1200, + "gen_cpp/segment_v2.pb.h": 1200, + "gen_cpp/PaloInternalService_types.h": 1150, + "util/threadpool.h": 600, # parsed_page.h holds RleDecoder by value and constructs it in the # header, so the whole storage read stack sees the RLE machinery; a live # dependency, budgeted as-is (trimming it is a refactor, not a gate). - "util/rle_encoding.h": 919, - "gen_cpp/internal_service.pb.h": 741, - "gen_cpp/FrontendService_types.h": 576, - "storage/options.h": 464, - "gen_cpp/cloud.pb.h": 342, - "gen_cpp/BackendService_types.h": 222, - "gen_cpp/data.pb.h": 220, - "io/fs/s3_file_system.h": 109, # the AWS SDK surface - "util/brpc_closure.h": 61, + "util/rle_encoding.h": 250, + "gen_cpp/internal_service.pb.h": 600, + "gen_cpp/FrontendService_types.h": 350, + "storage/options.h": 350, + "gen_cpp/cloud.pb.h": 50, + "gen_cpp/BackendService_types.h": 150, + "gen_cpp/data.pb.h": 120, + "io/fs/s3_file_system.h": 50, # the AWS SDK surface + "util/brpc_closure.h": 50, "runtime/workload_group/workload_group.h": 42, # thrift type universe carrier } REVERSE_SLACK = 0.10 From 09c0f1a768535c3cf79c0e48d3eb0351bfbaf23e Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Mon, 2 Nov 2026 20:48:52 +0800 Subject: [PATCH 3/3] [fix](be) Add direct includes for S3 reader tests and cache consumers ### What problem does this PR solve? Related PR: #69717 Problem Summary: Header dependency reductions left the S3 reader test and file-cache consumers without complete type definitions needed for object construction and member access. Include s3_file_system.h directly in the S3 reader test, storage/options.h in the file-cache HTTP action and microbench, and the rate limiter and FDCache definitions in the microbench. This restores independent compilation of the BE_TEST and enabled microbench paths. ### Release note None ### Check List (For Author) - Test: Standard ASAN BE/FE build; BE UT target build and 10 S3FileReaderTest/FileCacheActionTest cases with -j50 --coverage; independent compilation without PCH or Unity; build-hygiene; changed-line clang-format; clang-tidy; git diff --check. Coverage rendering reported 20 functions with mismatched profile data. - Behavior changed: No - Does this need documentation: No --- be/src/io/tools/file_cache_microbench.cpp | 3 +++ be/src/service/http/action/file_cache_action.cpp | 1 + be/test/io/fs/s3_file_reader_test.cpp | 1 + 3 files changed, 5 insertions(+) diff --git a/be/src/io/tools/file_cache_microbench.cpp b/be/src/io/tools/file_cache_microbench.cpp index 46dfb6934fd928..7e79f217e95cdd 100644 --- a/be/src/io/tools/file_cache_microbench.cpp +++ b/be/src/io/tools/file_cache_microbench.cpp @@ -47,10 +47,12 @@ #include "build/proto/microbench.pb.h" #include "common/config.h" #include "common/status.h" +#include "cpp/token_bucket_rate_limiter.h" #include "gflags/gflags.h" #include "io/cache/block_file_cache.h" #include "io/cache/block_file_cache_factory.h" #include "io/cache/cached_remote_file_reader.h" +#include "io/cache/fs_file_cache_storage.h" #include "io/file_factory.h" #include "io/fs/s3_file_system.h" #include "io/fs/s3_file_writer.h" @@ -59,6 +61,7 @@ #include "rapidjson/writer.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" +#include "storage/options.h" #include "storage/utils.h" #include "util/cpu_info.h" #include "util/disk_info.h" diff --git a/be/src/service/http/action/file_cache_action.cpp b/be/src/service/http/action/file_cache_action.cpp index 2403f970139f30..56ca7aa19d2b19 100644 --- a/be/src/service/http/action/file_cache_action.cpp +++ b/be/src/service/http/action/file_cache_action.cpp @@ -38,6 +38,7 @@ #include "service/http/http_request.h" #include "service/http/http_status.h" #include "storage/olap_define.h" +#include "storage/options.h" #include "storage/tablet/tablet_meta.h" #include "util/easy_json.h" diff --git a/be/test/io/fs/s3_file_reader_test.cpp b/be/test/io/fs/s3_file_reader_test.cpp index 6cb3579059993a..e8ef324ebd67ea 100644 --- a/be/test/io/fs/s3_file_reader_test.cpp +++ b/be/test/io/fs/s3_file_reader_test.cpp @@ -32,6 +32,7 @@ #include "common/config.h" #include "cpp/obj-client/s3_obj_storage_client.h" #include "io/fs/read_io_trace_test_util.h" +#include "io/fs/s3_file_system.h" #include "io/io_common.h" #include "runtime/runtime_profile.h" #include "util/countdown_latch.h"