Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions be/src/core/block/adaptive_block_size_predictor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <algorithm>
#include <cstddef>

#include "common/check.h"
#include "core/block/block.h"

namespace doris {
Expand All @@ -46,6 +47,12 @@ void AdaptiveBlockSizePredictor::update(const Block& block) {
}
}

void AdaptiveBlockSizePredictor::seed_history(double bytes_per_row) {
DORIS_CHECK(bytes_per_row > 0);
_bytes_per_row = bytes_per_row;
_has_history = true;
}

size_t AdaptiveBlockSizePredictor::predict_next_rows() const {
if (_block_size_bytes == 0) {
return _block_size_rows;
Expand Down
3 changes: 3 additions & 0 deletions be/src/core/block/adaptive_block_size_predictor.h
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ class AdaptiveBlockSizePredictor {

bool has_history() const { return _has_history; }

// Restore the measured row width shared by physical children of one source split.
void seed_history(double bytes_per_row);

private:
// EWMA weight for historical estimate (0.9) and current sample (0.1).
static constexpr double kAlpha = 0.9;
Expand Down
62 changes: 59 additions & 3 deletions be/src/exec/operator/file_scan_operator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,35 @@ ScannerScheduler* FileScanLocalState::scan_scheduler(RuntimeState* state) const
return state->get_query_ctx()->get_remote_scan_scheduler();
}

bool FileScanLocalState::_can_generate_physical_splits(bool is_load,
const TFileScanRangeParams& scan_params,
const TFileRangeDesc& range) {
if (is_load || (scan_params.__isset.table_format_params &&
scan_params.table_format_params.table_format_type == "transactional_hive")) {
return false;
}
const auto format = range.__isset.format_type ? range.format_type : scan_params.format_type;
if (format == TFileFormatType::FORMAT_PARQUET || format == TFileFormatType::FORMAT_ORC) {
// Keep scanner creation aligned with the downstream refinement guard. Otherwise an
// Iceberg delete split creates idle scanners even though it can never publish children.
return FileScannerV2::can_refine_source_split(range);
}
if (format != TFileFormatType::FORMAT_JNI || !range.__isset.table_format_params ||
range.table_format_params.table_format_type != "paimon" ||
!range.table_format_params.__isset.paimon_params) {
return false;
}
const auto& paimon = range.table_format_params.paimon_params;
return paimon.__isset.file_format &&
(paimon.file_format == "parquet" || paimon.file_format == "orc") &&
!paimon.__isset.paimon_split;
}

int FileScanLocalState::_adjust_scanner_count(int requested, int initial_ranges,
bool can_generate_physical_splits) {
return can_generate_physical_splits ? requested : std::min(requested, initial_ranges);
}

Status FileScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
if (_split_source->num_scan_ranges() == 0) {
_eos = true;
Expand Down Expand Up @@ -196,9 +225,36 @@ void FileScanLocalState::set_scan_ranges(RuntimeState* state,
if (_split_source == nullptr) {
_split_source = std::make_shared<LocalSplitSourceConnector>(scan_ranges, _max_scanners);
}
// currently the total number of splits in the bach split mode cannot be accurately obtained,
// so we don't do it in the batch split mode.
_max_scanners = std::min(_max_scanners, _split_source->num_scan_ranges());
// One FE columnar split can publish many format-local children after metadata planning.
// Keep the requested scanner concurrency so those children do not run serially.
bool can_generate_physical_splits = false;
const TFileScanRangeParams* common_params = nullptr;
if (state->get_query_ctx() != nullptr &&
state->get_query_ctx()->file_scan_range_params_map.contains(parent_id())) {
common_params = &state->get_query_ctx()->file_scan_range_params_map[parent_id()];
}
for (const auto& scan_range_params : scan_ranges) {
const auto& file_scan_range =
scan_range_params.scan_range.ext_scan_range.file_scan_range;
const auto* params =
file_scan_range.__isset.params ? &file_scan_range.params : common_params;
if (params == nullptr) {
continue;
}
const bool is_load =
state->desc_tbl().get_tuple_descriptor(params->src_tuple_id) != nullptr;
can_generate_physical_splits =
std::ranges::any_of(file_scan_range.ranges, [&](const auto& range) {
return _can_generate_physical_splits(is_load, *params, range);
});
if (can_generate_physical_splits) {
break;
}
}
// Currently the total number of remote splits cannot be accurately obtained, so batch
// mode already skips this cap.
_max_scanners = _adjust_scanner_count(_max_scanners, _split_source->num_scan_ranges(),
can_generate_physical_splits);
}

if (!scan_ranges.empty() &&
Expand Down
15 changes: 15 additions & 0 deletions be/src/exec/operator/file_scan_operator.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,17 @@ class FileScanLocalState final : public ScanLocalState<FileScanLocalState> {
int max_scanners_concurrency(RuntimeState* state) const override;
int min_scanners_concurrency(RuntimeState* state) const override;
ScannerScheduler* scan_scheduler(RuntimeState* state) const override;
#ifdef BE_TEST
static bool TEST_can_generate_physical_splits(bool is_load,
const TFileScanRangeParams& scan_params,
const TFileRangeDesc& range) {
return _can_generate_physical_splits(is_load, scan_params, range);
}
static int TEST_adjust_scanner_count(int requested, int initial_ranges,
bool can_generate_physical_splits) {
return _adjust_scanner_count(requested, initial_ranges, can_generate_physical_splits);
}
#endif

private:
friend class FileScannerV2;
Expand All @@ -67,6 +78,10 @@ class FileScanLocalState final : public ScanLocalState<FileScanLocalState> {
return PushDownType::PARTIAL_ACCEPTABLE;
}
bool _push_down_topn(const RuntimePredicate& predicate) override;
static bool _can_generate_physical_splits(bool is_load, const TFileScanRangeParams& scan_params,
const TFileRangeDesc& range);
static int _adjust_scanner_count(int requested, int initial_ranges,
bool can_generate_physical_splits);

PushDownType _should_push_down_is_null_predicate(VectorizedFnCall* fn_call) const override {
return fn_call->fn().name.function_name == "is_null_pred" ||
Expand Down
Loading
Loading