Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions be/src/exec/operator/file_scan_operator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include "core/data_type/data_type_struct.h"
#include "exec/operator/olap_scan_operator.h"
#include "exec/operator/scan_operator.h"
#include "exec/scan/file_scan_range_utils.h"
#include "exec/scan/file_scanner.h"
#include "exec/scan/file_scanner_v2.h"
#include "exec/scan/scanner_context.h"
Expand Down Expand Up @@ -188,8 +189,8 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o
// ADBC only has a FileScannerV2 reader, and enable_file_scanner_v2 is a session variable marked
// fuzzy=true, so the regression harness flips it to false at random. Without letting adbc
// through unconditionally, those queries land in v1, which has no "adbc" branch, and come back
// with an undiagnosable NotSupported -- showing up in CI as a random failure. This is the same
// mechanism as is_transactional_hive below, pointed the other way. Loads are left alone: there
// with an undiagnosable NotSupported -- showing up in CI as a random failure.
// Loads are left alone: there
// is no ADBC load path, so widening the rule to cover them would only route them somewhere they
// still cannot run.
// Fluss is in the same position, and reaches this the same way: FlussScanPlanProvider stamps the
Expand All @@ -202,9 +203,6 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o
scan_params.table_format_params.table_format_type == "fluss")) {
return true;
}
const bool is_transactional_hive =
scan_params.__isset.table_format_params &&
scan_params.table_format_params.table_format_type == "transactional_hive";
// Hybrid scans advertise native Parquet at scan level because ranges can arrive after the
// scanner is selected; FORMAT_JNI alone does not imply that every range uses JNI.
const bool is_hybrid_native_parquet = scan_params.format_type == TFileFormatType::FORMAT_JNI &&
Expand All @@ -224,7 +222,7 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o
// after scanner construction. Keep versioned Iceberg scans on V2 without eagerly listing
// every file.
return scanner_v2_requested && !is_load &&
scan_params.format_type != TFileFormatType::FORMAT_LANCE && !is_transactional_hive;
scan_params.format_type != TFileFormatType::FORMAT_LANCE;
}

Status FileScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
Expand All @@ -247,6 +245,7 @@ Status FileScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
} else {
scan_params = _split_source->get_params();
}
RETURN_IF_ERROR(validate_file_scan_support(*scan_params));
const bool is_load =
state()->desc_tbl().get_tuple_descriptor(scan_params->src_tuple_id) != nullptr;
// TODO: Use scanner v2 for all queries.
Expand Down
1 change: 0 additions & 1 deletion be/src/exec/rowid_fetcher.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,6 @@ TFileRangeDesc RowIdStorageReader::build_external_fetch_range(const TFileRangeDe
// needed by virtual columns, and do not mutate the FileMapping shared by other fetches.
auto range = source_range;
range.table_format_params.iceberg_params.__set_delete_files({});
range.table_format_params.transactional_hive_params = TTransactionalHiveDesc {};
return range;
}

Expand Down
28 changes: 28 additions & 0 deletions be/src/exec/scan/file_scan_range_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,38 @@

#pragma once

#include "common/status.h"
#include "gen_cpp/PlanNodes_types.h"

namespace doris {

inline Status validate_file_scan_support(const TFileScanRangeParams& params,
const TFileRangeDesc* range = nullptr) {
// Deprecated table markers must never fall through to plain file reads: that would lose
// Hive transaction visibility and LakeSoul merge semantics, including in metadata counts.
auto validate_table = [](const TTableFormatFileDesc& table) -> Status {
if (table.table_format_type == "transactional_hive") {
return Status::NotSupported("Hive transactional table reads are no longer supported");
}
if (table.table_format_type == "lakesoul") {
return Status::NotSupported("LakeSoul reads are no longer supported");
}
return Status::OK();
};
if (params.__isset.table_format_params) {
RETURN_IF_ERROR(validate_table(params.table_format_params));
}
if (range != nullptr && range->__isset.table_format_params) {
RETURN_IF_ERROR(validate_table(range->table_format_params));
}
const auto format = range != nullptr && range->__isset.format_type ? range->format_type
: params.format_type;
if (format == TFileFormatType::FORMAT_AVRO) {
return Status::NotSupported("Avro file reads are no longer supported");
}
return Status::OK();
}

inline bool can_ignore_not_found_file(const TFileRangeDesc& range, bool ignore_not_found) {
// Iceberg ranges belong to a committed snapshot, not a stale directory listing. Skipping a
// missing data or delete file would silently return an incomplete or incorrect snapshot.
Expand Down
19 changes: 4 additions & 15 deletions be/src/exec/scan/file_scanner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@
#include "format/table/paimon_reader.h"
#include "format/table/partition_column_filler.h"
#include "format/table/remote_doris_reader.h"
#include "format/table/transactional_hive_reader.h"
#include "format/table/trino_connector_jni_reader.h"
#include "format/text/text_reader.h"
#include "io/cache/block_file_cache_profile.h"
Expand Down Expand Up @@ -152,6 +151,7 @@ void FileScanner::_configure_file_scan_handlers() {
}

Status FileScanner::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) {
RETURN_IF_ERROR(validate_file_scan_support(*_params));
RETURN_IF_ERROR(Scanner::init(state, conjuncts));
_get_block_timer =
ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileScannerGetBlockTime", 1);
Expand Down Expand Up @@ -1030,6 +1030,7 @@ Status FileScanner::_get_next_reader() {
}

const TFileRangeDesc& range = _current_range;
RETURN_IF_ERROR(validate_file_scan_support(*_params, &range));
_current_range_path = range.path;

if (!_partition_slot_index_map.empty()) {
Expand Down Expand Up @@ -1456,20 +1457,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr,
octx.slot_id_to_filter_conjuncts = &_slot_id_to_filter_conjuncts;

if (range.__isset.table_format_params &&
range.table_format_params.table_format_type == "transactional_hive") {
// TransactionalHiveReader IS-A OrcReader, no wrapping needed
auto tran_orc_reader = TransactionalHiveReader::create_unique(
_profile, _state, *_params, range, _state->batch_size(), _state->timezone(),
_io_ctx, file_meta_cache_ptr);
tran_orc_reader->set_create_row_id_column_iterator_func(
[this]() -> std::shared_ptr<segment_v2::RowIdColumnIteratorV2> {
return _create_row_id_column_iterator();
});
init_status = static_cast<GenericReader*>(tran_orc_reader.get())->init_reader(&octx);

_cur_reader = std::move(tran_orc_reader);
} else if (range.__isset.table_format_params &&
range.table_format_params.table_format_type == "iceberg") {
range.table_format_params.table_format_type == "iceberg") {
// IcebergOrcReader IS-A OrcReader (CRTP mixin), no wrapping needed
std::unique_ptr<IcebergOrcReader> iceberg_reader = IcebergOrcReader::create_unique(
_kv_cache, _profile, _state, *_params, range, _state->batch_size(),
Expand Down Expand Up @@ -1587,6 +1575,7 @@ Status FileScanner::_generate_truncate_columns(bool need_to_get_parsed_schema) {
}

Status FileScanner::prepare_for_read_lines(const TFileRangeDesc& range) {
RETURN_IF_ERROR(validate_file_scan_support(*_params, &range));
_current_range = range;

_file_cache_statistics.reset(new io::FileCacheStatistics());
Expand Down
1 change: 1 addition & 0 deletions be/src/exec/scan/file_scanner_v2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,7 @@ bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFile

Status FileScannerV2::_validate_scan_range(const TFileScanRangeParams& params,
const TFileRangeDesc& range) {
RETURN_IF_ERROR(validate_file_scan_support(params, &range));
if (!is_supported(params, range)) {
return Status::NotSupported(
"FileScannerV2 does not support table format {} with file format {}",
Expand Down
Loading
Loading