diff --git a/be/src/exec/es/es_scan_reader.cpp b/be/src/exec/es/es_scan_reader.cpp index 18ffade1c29a31..f09538ba3e6e2b 100644 --- a/be/src/exec/es/es_scan_reader.cpp +++ b/be/src/exec/es/es_scan_reader.cpp @@ -19,6 +19,8 @@ #include +#include +#include #include #include #include @@ -28,6 +30,7 @@ #include "common/status.h" #include "exec/es/es_scroll_parser.h" #include "exec/es/es_scroll_query.h" +#include "service/backend_options.h" #include "service/http/http_method.h" namespace doris { @@ -266,4 +269,60 @@ Status ESScanReader::close() { return Status::InternalError("es_scan_reader delete scroll context failure"); } } + +std::string ESScanReader::extract_hostname(std::string host) { + auto scheme_end = host.find("://"); + if (scheme_end != std::string::npos) { + host = host.substr(scheme_end + 3); + } + + if (!host.empty() && host.front() == '[') { + auto bracket_end = host.find(']'); + if (bracket_end != std::string::npos) { + return host.substr(1, bracket_end - 1); + } + return host; + } + + auto colon = host.rfind(':'); + if (colon == std::string::npos) { + return host; + } + std::string port = host.substr(colon + 1); + if (!port.empty() && + std::ranges::all_of(port, [](unsigned char ch) { return std::isdigit(ch) != 0; })) { + return host.substr(0, colon); + } + return host; +} + +std::string ESScanReader::select_host(const std::map& properties) { + // If es_hosts contains multiple hosts, prefer localhost for locality + auto it = properties.find("es_hosts"); + if (it != properties.end() && !it->second.empty()) { + std::string localhost = BackendOptions::get_localhost(); + std::string best; + std::istringstream stream(it->second); + std::string host; + while (std::getline(stream, host, ',')) { + if (best.empty()) { + best = host; + } + std::string hostname = extract_hostname(host); + if (hostname == localhost) { + return host; + } + } + if (!best.empty()) { + return best; + } + } + // Fallback to host_port + auto hp = properties.find(ESScanReader::KEY_HOST_PORT); + if (hp != properties.end()) { + return hp->second; + } + return ""; +} + } // namespace doris diff --git a/be/src/exec/es/es_scan_reader.h b/be/src/exec/es/es_scan_reader.h index 1c8bd844f46dba..a980ac1beba067 100644 --- a/be/src/exec/es/es_scan_reader.h +++ b/be/src/exec/es/es_scan_reader.h @@ -58,6 +58,9 @@ class ESScanReader { // clear scroll context from elasticsearch Status close(); + static std::string extract_hostname(std::string host); + static std::string select_host(const std::map& properties); + private: std::string _target; std::string _user_name; diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index 60d26966864e16..5456c0bfa7172f 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -663,6 +663,17 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, std::vector& columns, bool* line_eof, const std::map& docvalue_context, const cctz::time_zone& time_zone) { + return fill_columns(tuple_desc->slots(), columns, line_eof, docvalue_context, time_zone); +} + +// Keep identical ES-to-Doris type dispatch for the tuple and file-local projection entry points. +// NOLINTNEXTLINE(readability-function-cognitive-complexity,readability-function-size) +Status ScrollParser::fill_columns(const std::vector& slots, + std::vector& columns, bool* line_eof, + const std::map& docvalue_context, + const cctz::time_zone& time_zone, + std::span column_positions) { + DORIS_CHECK(column_positions.empty() || column_positions.size() == slots.size()); *line_eof = true; if (_size <= 0 || _line_index >= _size) { @@ -682,9 +693,9 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, line = &obj["fields"]; } - for (int i = 0; i < tuple_desc->slots().size(); ++i) { - const SlotDescriptor* slot_desc = tuple_desc->slots()[i]; - auto* col_ptr = columns[i].get(); + for (size_t i = 0; i < slots.size(); ++i) { + const SlotDescriptor* slot_desc = slots[i]; + auto* col_ptr = columns[column_positions.empty() ? i : column_positions[i]].get(); if (slot_desc->col_name() == FIELD_ID) { // actually this branch will not be reached, this is guaranteed by Doris FE. @@ -888,10 +899,10 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, } case TYPE_ARRAY: { Array array; - const auto& sub_type = assert_cast( - remove_nullable(tuple_desc->slots()[i]->type()).get()) - ->get_nested_type() - ->get_primitive_type(); + const auto& sub_type = + assert_cast(remove_nullable(slot_desc->type()).get()) + ->get_nested_type() + ->get_primitive_type(); RETURN_IF_ERROR(parse_column(col, sub_type, pure_doc_value, array, time_zone)); col_ptr->insert(Field::create_field(array)); break; diff --git a/be/src/exec/es/es_scroll_parser.h b/be/src/exec/es/es_scroll_parser.h index b605bc2bddd5ab..e1f1106f624eb5 100644 --- a/be/src/exec/es/es_scroll_parser.h +++ b/be/src/exec/es/es_scroll_parser.h @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -30,6 +31,7 @@ namespace doris { class Status; class TupleDescriptor; +class SlotDescriptor; class ScrollParser { public: @@ -42,6 +44,13 @@ class ScrollParser { bool* line_eof, const std::map& docvalue_context, const cctz::time_zone& time_zone); + // Empty positions preserve tuple order; explicit positions address a projected file-local block. + Status fill_columns(const std::vector& slots, + std::vector& columns, bool* line_eof, + const std::map& docvalue_context, + const cctz::time_zone& time_zone, + std::span column_positions = {}); + const std::string& get_scroll_id(); int get_size() const; diff --git a/be/src/exec/operator/file_scan_operator.cpp b/be/src/exec/operator/file_scan_operator.cpp index 3a39de474f4c84..9ba2747624bcca 100644 --- a/be/src/exec/operator/file_scan_operator.cpp +++ b/be/src/exec/operator/file_scan_operator.cpp @@ -196,8 +196,7 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o // Iceberg's default write format does not describe retained files, and remote splits arrive // after scanner construction. Keep versioned Iceberg scans on V2 without eagerly listing // every file. - return scanner_v2_requested && !is_load && - scan_params.format_type != TFileFormatType::FORMAT_ES_HTTP && !is_transactional_hive; + return scanner_v2_requested && !is_load && !is_transactional_hive; } bool FileScanLocalState::_can_generate_physical_splits(const TQueryOptions& query_options, diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 18dc5a6ce49f70..45b40ee8ab41ee 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -59,6 +59,7 @@ #include "format_v2/jni/jdbc_reader.h" #include "format_v2/jni/max_compute_jni_reader.h" #include "format_v2/jni/trino_connector_jni_reader.h" +#include "format_v2/table/es_reader.h" #include "format_v2/table/hive_reader.h" #include "format_v2/table/hudi_reader.h" #include "format_v2/table/iceberg_position_delete_sys_table_reader.h" @@ -378,6 +379,8 @@ bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFile if (format_type == TFileFormatType::FORMAT_PARQUET || format_type == TFileFormatType::FORMAT_ORC) { return is_supported_table_format(range); + } else if (format_type == TFileFormatType::FORMAT_ES_HTTP) { + return table_format_name(range) == "es"; } else if (format_type == TFileFormatType::FORMAT_ARROW) { return is_supported_arrow_table_format(range); } else if (format_type == TFileFormatType::FORMAT_LANCE) { @@ -909,6 +912,8 @@ Status FileScannerV2::_create_table_reader_for_format( *reader = std::make_unique(mc_desc); } else if (table_format == "trino_connector") { *reader = std::make_unique(); + } else if (table_format == "es") { + *reader = std::make_unique(); } else if (table_format == "remote_doris") { *reader = std::make_unique(); } else { @@ -1233,6 +1238,9 @@ Status FileScannerV2::_to_file_format(TFileFormatType::type format_type, case TFileFormatType::FORMAT_ARROW: *file_format = format::FileFormat::ARROW; return Status::OK(); + case TFileFormatType::FORMAT_ES_HTTP: + *file_format = format::FileFormat::ES_HTTP; + return Status::OK(); case TFileFormatType::FORMAT_WAL: *file_format = format::FileFormat::WAL; return Status::OK(); diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index c1f11f2566f517..4e2e5e3afce28c 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -61,6 +61,7 @@ enum class FileFormat { ARROW, WAL, LANCE, + ES_HTTP, }; // Format-local description of one independently readable part of a file. Scanner scheduling diff --git a/be/src/format_v2/materialized_reader_util.cpp b/be/src/format_v2/materialized_reader_util.cpp index d27f066b30bc2b..ca2c4c86b43a93 100644 --- a/be/src/format_v2/materialized_reader_util.cpp +++ b/be/src/format_v2/materialized_reader_util.cpp @@ -21,7 +21,10 @@ #include "core/block/block.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" #include "core/types.h" #include "exprs/vexpr_context.h" #include "format_v2/file_reader.h" @@ -36,8 +39,52 @@ void update_counter(RuntimeProfile::Counter* counter, int64_t value) { } } +ColumnDefinition materialized_child_definition(const std::string& name, DataTypePtr type, + int32_t local_id) { + ColumnDefinition child; + child.identifier = Field::create_field(name); + child.local_id = local_id; + child.name = name; + child.type = std::move(type); + child.children = materialized_schema_children(child.type); + return child; +} + } // namespace +std::vector materialized_schema_children(const DataTypePtr& type) { + std::vector children; + DORIS_CHECK(type != nullptr); + const auto nested_type = remove_nullable(type); + switch (nested_type->get_primitive_type()) { + case TYPE_ARRAY: { + const auto* array_type = assert_cast(nested_type.get()); + children.push_back( + materialized_child_definition("element", array_type->get_nested_type(), 0)); + break; + } + case TYPE_MAP: { + const auto* map_type = assert_cast(nested_type.get()); + children.push_back(materialized_child_definition("key", map_type->get_key_type(), 0)); + children.push_back(materialized_child_definition("value", map_type->get_value_type(), 1)); + break; + } + case TYPE_STRUCT: { + const auto* struct_type = assert_cast(nested_type.get()); + children.reserve(struct_type->get_elements().size()); + for (size_t idx = 0; idx < struct_type->get_elements().size(); ++idx) { + children.push_back(materialized_child_definition(struct_type->get_element_name(idx), + struct_type->get_element(idx), + cast_set(idx))); + } + break; + } + default: + break; + } + return children; +} + ColumnPtr make_column_nullable_if_needed(ColumnPtr column, const DataTypePtr& target_type) { if (target_type != nullptr && target_type->is_nullable() && column.get() != nullptr && !column->is_nullable()) { diff --git a/be/src/format_v2/materialized_reader_util.h b/be/src/format_v2/materialized_reader_util.h index 2fb1383dfb9569..5f88ab061e7c74 100644 --- a/be/src/format_v2/materialized_reader_util.h +++ b/be/src/format_v2/materialized_reader_util.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include "common/status.h" #include "core/column/column.h" @@ -33,6 +34,10 @@ struct IOContext; namespace format { struct FileScanRequest; +struct ColumnDefinition; + +// Describe children of already materialized ARRAY/MAP/STRUCT values in their local schema. +std::vector materialized_schema_children(const DataTypePtr& type); // Shared helpers for FileReader implementations that deserialize or build already materialized // Doris columns and then hand those columns to TableReader for final mapping. diff --git a/be/src/format_v2/table/es_reader.cpp b/be/src/format_v2/table/es_reader.cpp new file mode 100644 index 00000000000000..3b495260768fb4 --- /dev/null +++ b/be/src/format_v2/table/es_reader.cpp @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/es_reader.h" + +#include + +#include "core/block/block.h" +#include "exec/es/es_scan_reader.h" +#include "exec/es/es_scroll_parser.h" +#include "exec/es/es_scroll_query.h" +#include "format_v2/column_mapper.h" +#include "format_v2/materialized_reader_util.h" +#include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" +#include "runtime/runtime_state.h" + +namespace doris::format::es { +namespace { + +Status validate_es_range(const TFileRangeDesc& range) { + if (!range.__isset.table_format_params || range.table_format_params.table_format_type != "es" || + !range.table_format_params.__isset.es_params) { + return Status::InvalidArgument("ES v2 reader requires es table format and es_params"); + } + return Status::OK(); +} + +class HttpScrollStream final : public EsScrollStream { +public: + HttpScrollStream(const std::string& host, const std::map& properties, + bool docvalues) + : _reader(host, properties, docvalues) {} + Status open() override { return _reader.open(); } + Status next(bool* eof, std::unique_ptr& parser) override { + return _reader.get_next(eof, parser); + } + Status close() override { return _reader.close(); } + +private: + ESScanReader _reader; +}; + +} // namespace + +EsFileReader::EsFileReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + TFileRangeDesc range, const TFileScanRangeParams& params, + std::vector file_slot_descs, + EsScrollStreamFactory stream_factory) + : FileReader(system_properties, file_description, std::move(io_ctx), profile), + _range(std::move(range)), + _params(params), + _file_slots(std::move(file_slot_descs)), + _stream_factory(std::move(stream_factory)) {} + +EsFileReader::~EsFileReader() { + WARN_IF_ERROR(close(), "Failed to close ES scroll stream"); +} + +void EsFileReader::_init_profile() { + if (_profile == nullptr) { + return; + } + _io_time = file_scan_profile::ensure_hierarchy(_profile).io; + const char* parent = "EsFileReader"; + _total_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, parent, file_scan_profile::FILE_READER, 1); + _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "EsReadTime", parent, 1); + _materialize_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "EsMaterializeTime", parent, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "EsFilterTime", parent, 1); + _batches_read = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "EsBatchesRead", TUnit::UNIT, parent, 1); + _rows_read = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "EsRowsRead", TUnit::UNIT, parent, 1); +} + +Status EsFileReader::init(RuntimeState* state) { + DORIS_CHECK(state != nullptr); + _state = state; + set_batch_size(state->batch_size()); + _init_profile(); + RETURN_IF_ERROR(validate_es_range(_range)); + _eof = false; + return Status::OK(); +} + +Status EsFileReader::get_schema(std::vector* schema) const { + DORIS_CHECK(schema != nullptr); + schema->clear(); + schema->reserve(_file_slots.size()); + for (size_t i = 0; i < _file_slots.size(); ++i) { + const auto* slot = _file_slots[i]; + schema->push_back({.identifier = Field::create_field(cast_set(i)), + .local_id = cast_set(i), + .name = slot->col_name(), + .type = slot->type(), + .children = materialized_schema_children(slot->type())}); + } + return Status::OK(); +} + +std::unique_ptr EsFileReader::create_column_mapper( + TableColumnMapperOptions options) const { + return std::make_unique(options); +} + +void EsFileReader::set_batch_size(size_t batch_size) { + DORIS_CHECK(batch_size > 0); + _batch_size = batch_size; +} + +Status EsFileReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + RETURN_IF_ERROR(FileReader::open(std::move(request))); + auto status = _open_stream(); + if (!status.ok()) { + WARN_IF_ERROR(close(), "Failed to clear ES scroll after open error"); + } + return status; +} + +Status EsFileReader::_open_stream() { + DORIS_CHECK(_stream == nullptr); + RETURN_IF_CANCELLED(_state); + if (_io_ctx != nullptr && _io_ctx->should_stop) { + _eof = true; + return Status::OK(); + } + std::vector names; + for (const auto& [id, position] : _request->local_positions) { + DORIS_CHECK(id.value() >= 0 && id.value() < _file_slots.size()); + _read_slots.push_back(_file_slots[id.value()]); + _read_positions.push_back(position.value()); + names.push_back(_file_slots[id.value()]->col_name()); + } + auto properties = _params.es_properties; + for (const auto& [key, value] : _range.table_format_params.es_params) { + properties[key] = value; + } + properties[ESScanReader::KEY_BATCH_SIZE] = std::to_string(_batch_size); + _docvalue_context = _params.es_docvalue_context; + // A source-only field such as _id cannot use doc values, even when other requested fields can. + // The mapper may retain hidden predicate columns beyond the FE's original output projection. + for (const auto& name : names) { + if (name == "_id" || !_docvalue_context.contains(name)) { + properties[ESScanReader::KEY_DOC_VALUES_MODE] = "0"; + break; + } + } + const auto host = ESScanReader::select_host(properties); + for (const auto* key : + {ESScanReader::KEY_INDEX, ESScanReader::KEY_SHARD, ESScanReader::KEY_QUERY_DSL}) { + if (!properties.contains(key) || properties.at(key).empty()) { + return Status::InvalidArgument("ES v2 reader requires property {}", key); + } + } + if (host.empty()) { + return Status::InvalidArgument("ES v2 reader requires a host"); + } + properties[ESScanReader::KEY_HOST_PORT] = host; + bool docvalues = false; + properties[ESScanReader::KEY_QUERY] = + ESScrollQueryBuilder::build(properties, names, _docvalue_context, &docvalues); + _stream = _stream_factory ? _stream_factory(host, properties, docvalues) + : std::make_unique(host, properties, docvalues); + DORIS_CHECK(_stream != nullptr); + { + SCOPED_TIMER(_io_time); + SCOPED_TIMER(_read_time); + RETURN_IF_ERROR(_stream->open()); + } + // ESScanReader caches the first response in open(). Parse it now to capture the scroll ID so + // LIMIT/cancellation before the first block can still clear the server-side search context. + RETURN_IF_ERROR(_next_page()); + _eof = false; + return Status::OK(); +} + +Status EsFileReader::_next_page() { + SCOPED_TIMER(_io_time); + SCOPED_TIMER(_read_time); + RETURN_IF_ERROR(_stream->next(&_source_eof, _parser)); + _page_rows_remaining = _source_eof ? 0 : _parser->get_size(); + if (!_source_eof && _batches_read != nullptr) { + COUNTER_UPDATE(_batches_read, 1); + } + return Status::OK(); +} + +Status EsFileReader::get_block(Block* block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); + DORIS_CHECK(block != nullptr && rows != nullptr && eof != nullptr); + *rows = 0; + *eof = false; + if (_eof) { + *eof = true; + return Status::OK(); + } + auto status = _read_block(block, rows, eof); + if (!status.ok()) { + WARN_IF_ERROR(close(), "Failed to clear ES scroll after read error"); + return status; + } + if (*eof) { + return close(); + } + return Status::OK(); +} + +Status EsFileReader::_read_block(Block* block, size_t* rows, bool* eof) { + { + auto guard = block->mutate_columns_scoped(); + auto& columns = guard.mutable_columns(); + while (*rows < _batch_size) { + RETURN_IF_CANCELLED(_state); + if (_io_ctx != nullptr && _io_ctx->should_stop) { + *eof = true; + return Status::OK(); + } + if (_page_rows_remaining == 0) { + if (_source_eof) { + break; + } + RETURN_IF_ERROR(_next_page()); + if (_page_rows_remaining == 0) { + continue; + } + } + SCOPED_TIMER(_materialize_time); + bool line_eof = false; + // The parser sees physical slots only; the request owns their file-local block order. + RETURN_IF_ERROR(_parser->fill_columns(_read_slots, columns, &line_eof, + _docvalue_context, _state->timezone_obj(), + _read_positions)); + DORIS_CHECK(!line_eof); + --_page_rows_remaining; + ++*rows; + } + } + _record_scan_rows(cast_set(*rows)); + if (_rows_read != nullptr) { + COUNTER_UPDATE(_rows_read, *rows); + } + *eof = _source_eof && *rows == 0; + SCOPED_TIMER(_filter_time); + return apply_materialized_reader_filters(_request.get(), _io_ctx.get(), block, rows); +} + +Status EsFileReader::close() { + auto stream = std::move(_stream); + _parser.reset(); + _request.reset(); + _read_slots.clear(); + _read_positions.clear(); + _page_rows_remaining = 0; + _source_eof = true; + _eof = true; + // Release ownership before closing so a failed DELETE is reported once, including destruction. + return stream != nullptr ? stream->close() : Status::OK(); +} + +EsReader::EsReader(EsScrollStreamFactory stream_factory) + : _stream_factory(std::move(stream_factory)) {} + +Status EsReader::init(TableReadOptions&& options) { + DORIS_CHECK(options.file_slot_descs != nullptr && options.scan_params != nullptr); + return TableReader::init(std::move(options)); +} + +Status EsReader::prepare_split(const SplitReadOptions& options) { + RETURN_IF_ERROR(validate_es_range(options.current_range)); + return TableReader::prepare_split(options); +} + +Status EsReader::create_file_reader(std::unique_ptr* reader) { + *reader = std::make_unique(_system_properties, _current_task->data_file, _io_ctx, + _scanner_profile, _current_file_range_desc, + *_scan_params, *_file_slot_descs, _stream_factory); + return Status::OK(); +} + +} // namespace doris::format::es diff --git a/be/src/format_v2/table/es_reader.h b/be/src/format_v2/table/es_reader.h new file mode 100644 index 00000000000000..fea1869ff020e3 --- /dev/null +++ b/be/src/format_v2/table/es_reader.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "format_v2/file_reader.h" +#include "format_v2/table_reader.h" + +namespace doris { +class ScrollParser; +class SlotDescriptor; +} // namespace doris + +namespace doris::format::es { + +// The transport boundary allows reader tests to use real ES responses without a running server. +class EsScrollStream { +public: + virtual ~EsScrollStream() = default; + virtual Status open() = 0; + virtual Status next(bool* eof, std::unique_ptr& parser) = 0; + virtual Status close() = 0; +}; +using EsScrollStreamFactory = std::function( + const std::string&, const std::map&, bool)>; + +class EsFileReader final : public FileReader { +public: + EsFileReader(std::shared_ptr& system_properties, + std::unique_ptr& file_description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + TFileRangeDesc range, const TFileScanRangeParams& params, + std::vector file_slot_descs, + EsScrollStreamFactory stream_factory = {}); + ~EsFileReader() override; + + Status init(RuntimeState* state) override; + Status get_schema(std::vector* schema) const override; + std::unique_ptr create_column_mapper( + TableColumnMapperOptions options) const override; + void set_batch_size(size_t batch_size) override; + Status open(std::shared_ptr request) override; + Status get_block(Block* block, size_t* rows, bool* eof) override; + Status close() override; + +private: + void _init_profile() override; + Status _open_stream(); + Status _next_page(); + Status _read_block(Block* block, size_t* rows, bool* eof); + + const TFileRangeDesc _range; + const TFileScanRangeParams& _params; + const std::vector _file_slots; + EsScrollStreamFactory _stream_factory; + RuntimeState* _state = nullptr; + size_t _batch_size = 1024; + std::unique_ptr _stream; + std::unique_ptr _parser; + std::vector _read_slots; + std::vector _read_positions; + std::map _docvalue_context; + size_t _page_rows_remaining = 0; + bool _source_eof = false; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _read_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; + RuntimeProfile::Counter* _io_time = nullptr; + RuntimeProfile::Counter* _batches_read = nullptr; + RuntimeProfile::Counter* _rows_read = nullptr; +}; + +class EsReader final : public TableReader { +public: + explicit EsReader(EsScrollStreamFactory stream_factory = {}); + Status init(TableReadOptions&& options) override; + Status prepare_split(const SplitReadOptions& options) override; + +protected: + Status create_file_reader(std::unique_ptr* reader) override; + +private: + EsScrollStreamFactory _stream_factory; +}; + +} // namespace doris::format::es diff --git a/be/src/format_v2/table/remote_doris_reader.cpp b/be/src/format_v2/table/remote_doris_reader.cpp index bd2c08b5f2310f..c119275fc69f96 100644 --- a/be/src/format_v2/table/remote_doris_reader.cpp +++ b/be/src/format_v2/table/remote_doris_reader.cpp @@ -34,10 +34,7 @@ #include "core/assert_cast.h" #include "core/block/block.h" #include "core/data_type/data_type.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_struct.h" #include "core/data_type_serde/data_type_serde.h" #include "format/arrow/arrow_utils.h" #include "format_v2/materialized_reader_util.h" @@ -271,53 +268,6 @@ Status create_flight_stream(const TFileRangeDesc& range, std::shared_ptr synthesize_remote_doris_children(const DataTypePtr& type) { - std::vector children; - DORIS_CHECK(type != nullptr); - const auto nested_type = remove_nullable(type); - switch (nested_type->get_primitive_type()) { - case TYPE_ARRAY: { - const auto* array_type = assert_cast(nested_type.get()); - children.push_back( - remote_doris_child_definition("element", array_type->get_nested_type(), 0)); - break; - } - case TYPE_MAP: { - const auto* map_type = assert_cast(nested_type.get()); - children.push_back(remote_doris_child_definition("key", map_type->get_key_type(), 0)); - children.push_back(remote_doris_child_definition("value", map_type->get_value_type(), 1)); - break; - } - case TYPE_STRUCT: { - const auto* struct_type = assert_cast(nested_type.get()); - children.reserve(struct_type->get_elements().size()); - for (size_t idx = 0; idx < struct_type->get_elements().size(); ++idx) { - children.push_back(remote_doris_child_definition(struct_type->get_element_name(idx), - struct_type->get_element(idx), - cast_set(idx))); - } - break; - } - default: - break; - } - return children; -} - -ColumnDefinition remote_doris_child_definition(const std::string& name, DataTypePtr type, - int32_t local_id) { - ColumnDefinition child; - child.identifier = Field::create_field(name); - child.local_id = local_id; - child.name = name; - child.type = std::move(type); - child.children = synthesize_remote_doris_children(child.type); - return child; -} - } // namespace RemoteDorisFileReader::RemoteDorisFileReader( @@ -383,7 +333,7 @@ Status RemoteDorisFileReader::get_schema(std::vector* file_sch .type = slot->type(), // Remote Doris exposes table slots as file columns. Complex columns still need // structural children so TableColumnMapper can validate and project them. - .children = synthesize_remote_doris_children(slot->type()), + .children = materialized_schema_children(slot->type()), }); } return Status::OK(); diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 39ec11ac26b8e7..4750aef67f8ae0 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -206,6 +206,8 @@ std::string file_format_to_string(FileFormat format) { return "JNI"; case FileFormat::ARROW: return "ARROW"; + case FileFormat::ES_HTTP: + return "ES_HTTP"; case FileFormat::WAL: return "WAL"; case FileFormat::LANCE: diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index fba2acb97cdc4d..a86c665f41bde1 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -355,6 +355,10 @@ TEST(FileScannerV2Test, SupportedFormatMatrix) { {"hive", TFileFormatType::FORMAT_ARROW, std::nullopt, false}, {"", TFileFormatType::FORMAT_ARROW, std::nullopt, false}, {"", TFileFormatType::FORMAT_WAL, std::nullopt, true}, + {.table_format = "es", + .params_format = TFileFormatType::FORMAT_ES_HTTP, + .range_format = std::nullopt, + .expected = true}, {"", TFileFormatType::FORMAT_ES_HTTP, std::nullopt, false}, {"", TFileFormatType::FORMAT_LANCE, std::nullopt, false}, }; @@ -446,12 +450,12 @@ TEST(FileScannerV2Test, FileScanLocalStateSelectsV2ForSupportedQueriesAndLoads) params.__set_format_type(TFileFormatType::FORMAT_LANCE); EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); - const std::vector unsupported_formats {TFileFormatType::FORMAT_ES_HTTP}; - for (const auto format : unsupported_formats) { - params.__set_format_type(format); - EXPECT_FALSE( - FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); - } + params.__set_format_type(TFileFormatType::FORMAT_ES_HTTP); + EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); + EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, true, params)); + query_options.__set_enable_file_scanner_v2(false); + EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); + query_options.__set_enable_file_scanner_v2(true); params.__set_format_type(TFileFormatType::FORMAT_ORC); TTableFormatFileDesc table_format_params; diff --git a/be/test/format_v2/table/es_reader_test.cpp b/be/test/format_v2/table/es_reader_test.cpp new file mode 100644 index 00000000000000..ba25e745ad2c1d --- /dev/null +++ b/be/test/format_v2/table/es_reader_test.cpp @@ -0,0 +1,543 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/es_reader.h" + +#include +#include + +#include +#include +#include + +#include "common/object_pool.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "exec/es/es_scroll_parser.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "format_v2/column_mapper.h" +#include "runtime/runtime_state.h" +#include "service/http/ev_http_server.h" +#include "service/http/http_channel.h" +#include "service/http/http_handler.h" +#include "service/http/http_request.h" +#include "testutil/desc_tbl_builder.h" + +namespace doris::format::es { +namespace { +struct StreamState { + int opens = 0; + int reads = 0; + int closes = 0; + bool fail_open = false; + bool fail_next = false; + bool fail_close = false; + bool docvalues = false; + std::string host; + std::map properties; + std::vector pages; +}; +class ResponseStream final : public EsScrollStream { +public: + explicit ResponseStream(std::shared_ptr state) : _state(std::move(state)) {} + Status open() override { + ++_state->opens; + return _state->fail_open ? Status::IOError("open failed") : Status::OK(); + } + Status next(bool* eof, std::unique_ptr& parser) override { + ++_state->reads; + if (_state->fail_next) { + return Status::IOError("scroll failed"); + } + *eof = _page == _state->pages.size(); + if (!*eof) { + parser = std::make_unique(_state->docvalues); + RETURN_IF_ERROR(parser->parse(_state->pages[_page++])); + } + return Status::OK(); + } + Status close() override { + ++_state->closes; + return _state->fail_close ? Status::IOError("close failed") : Status::OK(); + } + +private: + std::shared_ptr _state; + size_t _page = 0; +}; + +std::string page(const std::string& hits) { + return R"({"_scroll_id":"scroll-token","hits":{"hits":[)" + hits + "]}}"; +} +TFileRangeDesc es_range() { + TTableFormatFileDesc table; + table.__set_table_format_type("es"); + table.__set_es_params( + {{"index", "example"}, {"shard_id", "0"}, {"host_port", "http://localhost:9200"}}); + TFileRangeDesc range; + range.__set_format_type(TFileFormatType::FORMAT_ES_HTTP); + range.__set_path("es://example/0"); + range.__set_table_format_params(table); + return range; +} +EsScrollStreamFactory factory(const std::shared_ptr& state) { + return [state](const auto& host, const auto& properties, bool docvalues) { + state->host = host; + state->properties = properties; + state->docvalues = docvalues; + return std::make_unique(state); + }; +} +class EsV2ReaderTest : public testing::Test { +protected: + void SetUp() override { + DescriptorTblBuilder builder(&_pool); + builder.declare_tuple() + << std::make_tuple(std::make_shared(), std::string("id")) + << std::make_tuple(std::make_shared(), std::string("name")) + << std::make_tuple(std::make_shared(), std::string("_id")) + << std::make_tuple(std::make_shared( + make_nullable(std::make_shared())), + std::string("values")); + _slots = builder.build()->get_tuple_descriptor(0)->slots(); + _params.__set_es_properties({{"query_dsl", R"({"match_all":{}})"}}); + _responses = std::make_shared(); + _responses->pages = { + page(R"({"_id":"a","_source":{"id":1,"name":"one","values":[1,2,3]}}, + {"_id":"b","_source":{"id":2,"name":null,"values":[]}})"), + page(R"({"_id":"c","_source":{"id":3,"name":"three","values":null}})")}; + } + std::unique_ptr reader() { + auto props = std::make_shared(); + auto file = std::make_unique(); + file->path = "es://example/0"; + auto result = std::make_unique(props, file, _io, &_profile, _range, _params, + _slots, factory(_responses)); + EXPECT_TRUE(result->init(&_state).ok()); + return result; + } + std::shared_ptr request(std::initializer_list ids) { + auto result = std::make_shared(); + FileScanRequestBuilder builder(result.get()); + for (int id : ids) { + EXPECT_TRUE(builder.add_non_predicate_column(LocalColumnId(id)).ok()); + } + return result; + } + Block block(const EsFileReader& reader, const FileScanRequest& request) { + std::vector schema; + EXPECT_TRUE(reader.get_schema(&schema).ok()); + ColumnsWithTypeAndName columns(request.block_column_count()); + for (auto [id, position] : request.local_positions) { + const auto& column = schema[id.value()]; + columns[position.value()] = {column.type->create_column(), column.type, column.name}; + } + return {std::move(columns)}; + } + int32_t int_at(const Block& block, size_t pos, size_t row) { + const auto& nullable = + assert_cast(*block.get_by_position(pos).column); + return assert_cast(nullable.get_nested_column()).get_data()[row]; + } + std::string string_at(const Block& block, size_t pos, size_t row) { + const auto& nullable = + assert_cast(*block.get_by_position(pos).column); + return nullable.get_nested_column().get_data_at(row).to_string(); + } + ObjectPool _pool; + RuntimeState _state; + RuntimeProfile _profile {"es_v2_test"}; + std::shared_ptr _io = std::make_shared(); + TFileRangeDesc _range = es_range(); + TFileScanRangeParams _params; + std::vector _slots; + std::shared_ptr _responses; +}; +class NullableIntGreaterThanExpr final : public VExpr { +public: + NullableIntGreaterThanExpr(size_t block_position, int32_t value) + : VExpr(std::make_shared(), false), + _block_position(block_position), + _value(value) {} + + const std::string& expr_name() const override { return _name; } + + bool is_constant() const override { return false; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& nullable = + assert_cast(*block->get_by_position(_block_position).column); + const auto& data = assert_cast(nullable.get_nested_column()); + + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const auto source_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = + !nullable.is_null_at(source_row) && data.get_element(source_row) > _value; + } + result_column = std::move(result); + return Status::OK(); + } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_block_position, _value); + return Status::OK(); + } + +private: + size_t _block_position; + int32_t _value; + const std::string _name = "NullableIntGreaterThanExpr"; +}; + +VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) { + auto context = VExprContext::create_shared(expr); + auto status = context->prepare(state, RowDescriptor()); + EXPECT_TRUE(status.ok()) << status; + status = context->open(state); + EXPECT_TRUE(status.ok()) << status; + return context; +} + +} // namespace + +// NOLINTNEXTLINE(readability-function-cognitive-complexity): GoogleTest assertions expand to control flow. +TEST_F(EsV2ReaderTest, ProjectsLocalColumnsAndCarriesPagesAcrossBatches) { + auto scan = reader(); + std::vector schema; + ASSERT_TRUE(scan->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 4); + ASSERT_EQ(schema[3].children.size(), 1); + EXPECT_EQ(_responses->opens, 0); + auto projection = request({1, 0, 2, 3}); + ASSERT_TRUE(scan->open(projection).ok()); + EXPECT_EQ(_responses->reads, 1); + scan->set_batch_size(1); + size_t rows = 0; + bool eof = false; + for (size_t row = 0; row < 3; ++row) { + auto output = block(*scan, *projection); + auto status = scan->get_block(&output, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows, 1); + EXPECT_FALSE(eof); + EXPECT_EQ(int_at(output, 1, 0), row + 1); + EXPECT_EQ(string_at(output, 2, 0), std::string(1, 'a' + row)); + if (row == 1) { + EXPECT_TRUE(output.get_by_position(0).column->is_null_at(0)); + } else { + EXPECT_EQ(string_at(output, 0, 0), row == 0 ? "one" : "three"); + } + if (row == 0) { + const auto& values = assert_cast( + assert_cast(*output.get_by_position(3).column) + .get_nested_column()); + EXPECT_EQ(values.get_offsets()[0], 3); + EXPECT_FALSE(values.get_data().is_null_at(1)); + } + } + auto output = block(*scan, *projection); + ASSERT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + EXPECT_EQ(rows, 0); + EXPECT_TRUE(eof); + EXPECT_EQ(_responses->closes, 1); + ASSERT_TRUE(scan->close().ok()); + scan.reset(); + EXPECT_EQ(_responses->closes, 1); +} + +TEST_F(EsV2ReaderTest, UsesDocvalueAliasesAndRangeOverrides) { + _params.es_properties["index"] = "node-index"; + _params.es_properties["doc_values_mode"] = "1"; + _params.__set_es_docvalue_context({{"id", "id"}, {"name", "name.keyword"}}); + _responses->pages = {page(R"({"fields":{"id":[4],"name.keyword":["four"]}})")}; + auto scan = reader(); + auto projection = request({1, 0}); + ASSERT_TRUE(scan->open(projection).ok()); + EXPECT_TRUE(_responses->docvalues); + EXPECT_EQ(_responses->properties.at("index"), "example"); + EXPECT_NE(_responses->properties.at("query").find("name.keyword"), std::string::npos); + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(int_at(output, 1, 0), 4); + EXPECT_EQ(string_at(output, 0, 0), "four"); +} + +TEST_F(EsV2ReaderTest, CountsRowsWithNoProjectedColumns) { + auto scan = reader(); + auto projection = request({}); + ASSERT_TRUE(scan->open(projection).ok()); + Block output; + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + EXPECT_EQ(rows, 3); + ASSERT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); +} + +TEST_F(EsV2ReaderTest, AppliesResidualFiltersAndAccountsForFilteredRows) { + auto scan = reader(); + auto projection = request({1, 0}); + projection->conjuncts.push_back( + prepared_conjunct(&_state, std::make_shared(1, 1))); + ASSERT_TRUE(scan->open(projection).ok()); + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + auto status = scan->get_block(&output, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(rows, 2); + EXPECT_EQ(int_at(output, 1, 0), 2); + EXPECT_EQ(int_at(output, 1, 1), 3); + EXPECT_EQ(_io->predicate_filtered_rows, 1); +} + +TEST_F(EsV2ReaderTest, ClosesAfterOpenFailureAndBeforeFirstBlock) { + for (int mode = 0; mode < 3; ++mode) { + _responses = std::make_shared(); + _responses->pages = {page("")}; + _responses->fail_open = mode == 0; + _responses->fail_next = mode == 1; + auto scan = reader(); + auto status = scan->open(request({0})); + EXPECT_EQ(status.ok(), mode == 2); + if (mode != 2) { + EXPECT_EQ(_responses->closes, 1); + } + ASSERT_TRUE(scan->close().ok()); + scan.reset(); + EXPECT_EQ(_responses->closes, 1); + } +} + +TEST_F(EsV2ReaderTest, CancellationAndReadErrorsReleaseStream) { + for (bool fail_next : {false, true}) { + _io->should_stop = false; + _responses = std::make_shared(); + _responses->pages = {page(R"({"_source":{"id":1}})")}; + auto scan = reader(); + auto projection = request({0}); + ASSERT_TRUE(scan->open(projection).ok()); + _responses->fail_next = fail_next; + _io->should_stop = !fail_next; + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + auto status = scan->get_block(&output, &rows, &eof); + EXPECT_EQ(status.ok(), !fail_next); + EXPECT_EQ(_responses->closes, 1); + ASSERT_TRUE(scan->close().ok()); + EXPECT_EQ(_responses->closes, 1); + } +} + +TEST_F(EsV2ReaderTest, CloseFailureIsReportedOnce) { + auto scan = reader(); + ASSERT_TRUE(scan->open(request({0})).ok()); + _responses->fail_close = true; + EXPECT_FALSE(scan->close().ok()); + ASSERT_TRUE(scan->close().ok()); + scan.reset(); + EXPECT_EQ(_responses->closes, 1); +} + +TEST_F(EsV2ReaderTest, RejectsMissingRangePropertiesAndRowidFetch) { + _range.table_format_params.__isset.es_params = false; + auto props = std::make_shared(); + auto file = std::make_unique(); + EsFileReader invalid(props, file, _io, &_profile, _range, _params, _slots, factory(_responses)); + EXPECT_FALSE(invalid.init(&_state).ok()); + _range = es_range(); + auto scan = reader(); + auto projection = request({0}); + projection->row_ids = std::vector {0}; + EXPECT_FALSE(scan->open(projection).ok()); + EXPECT_EQ(_responses->opens, 0); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity): GoogleTest assertions expand to control flow. +TEST_F(EsV2ReaderTest, TableReaderMapsProjectionAndAdvancesSplits) { + EsReader table(factory(_responses)); + std::vector projected; + for (int id : {1, 0}) { + ColumnDefinition column; + column.identifier = Field::create_field(_slots[id]->col_name()); + column.name = _slots[id]->col_name(); + column.type = _slots[id]->type(); + projected.push_back(std::move(column)); + } + ColumnDefinition missing; + missing.name = "missing"; + missing.type = make_nullable(std::make_shared()); + projected.push_back(std::move(missing)); + ASSERT_TRUE(table.init({.projected_columns = projected, + .conjuncts = {}, + .format = FileFormat::ES_HTTP, + .scan_params = &_params, + .io_ctx = _io, + .runtime_state = &_state, + .scanner_profile = &_profile, + .file_slot_descs = &_slots}) + .ok()); + for (int split = 0; split < 2; ++split) { + SplitReadOptions options; + options.current_range = _range; + options.current_split_format = FileFormat::ES_HTTP; + ASSERT_TRUE(table.prepare_split(options).ok()); + Block output; + for (const auto& column : projected) { + output.insert({column.type->create_column(), column.type, column.name}); + } + bool eof = false; + size_t total = 0; + while (!eof) { + const auto status = table.get_block(&output, &eof); + ASSERT_TRUE(status.ok()) << status; + if (output.rows() > 0) { + ASSERT_EQ(output.columns(), 3); + EXPECT_EQ(string_at(output, 0, 0), "one"); + EXPECT_EQ(int_at(output, 1, 0), 1); + EXPECT_TRUE(output.get_by_position(2).column->is_null_at(0)); + total += output.rows(); + } + } + EXPECT_EQ(total, 3); + EXPECT_EQ(_responses->closes, split + 1); + } + ASSERT_TRUE(table.close().ok()); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity): GoogleTest assertions expand to control flow. +TEST_F(EsV2ReaderTest, RealHttpTransportClearsLatestScrollId) { + class Handler final : public HttpHandler { + public: + void handle(HttpRequest* req) override { + if (req->method() == DELETE) { + EXPECT_EQ(req->get_request_body(), R"({"scroll_id":"last-scroll"})"); + ++clears; + HttpChannel::send_reply(req, HttpStatus::OK, R"({"succeeded":true})"); + return; + } + const int call = searches.fetch_add(1); + const std::string response = + call < 2 + ? R"({"_scroll_id":"scroll-page","hits":{"hits":[{"_source":{"id":7}}]}})" + : R"({"_scroll_id":"last-scroll","hits":{"hits":[]}})"; + HttpChannel::send_reply(req, HttpStatus::OK, response); + } + std::atomic searches {0}; + std::atomic clears {0}; + } handler; + EvHttpServer server {0}; + ASSERT_TRUE(server.register_handler(POST, "/example/_search", &handler)); + ASSERT_TRUE(server.register_handler(POST, "/_search/scroll", &handler)); + ASSERT_TRUE(server.register_handler(DELETE, "/_search/scroll", &handler)); + server.start(); + ASSERT_GT(server.get_real_port(), 0); + _range.table_format_params.es_params["host_port"] = + "http://127.0.0.1:" + std::to_string(server.get_real_port()); + auto props = std::make_shared(); + auto file = std::make_unique(); + EsFileReader scan(props, file, _io, &_profile, _range, _params, _slots); + ASSERT_TRUE(scan.init(&_state).ok()); + scan.set_batch_size(1); + auto projection = request({0}); + const auto open_status = scan.open(projection); + ASSERT_TRUE(open_status.ok()) << open_status; + size_t rows = 0; + bool eof = false; + size_t total = 0; + while (!eof) { + auto output = block(scan, *projection); + const auto status = scan.get_block(&output, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + if (rows > 0) { + EXPECT_EQ(int_at(output, 0, 0), 7); + } + total += rows; + } + EXPECT_EQ(total, 2); + EXPECT_EQ(handler.searches.load(), 3); + EXPECT_EQ(handler.clears.load(), 1); + ASSERT_TRUE(scan.close().ok()); + EXPECT_EQ(handler.clears.load(), 1); +} + +TEST_F(EsV2ReaderTest, SparseProjectionDoesNotRequestOtherFields) { + auto scan = reader(); + auto projection = request({2}); + ASSERT_TRUE(scan->open(projection).ok()); + EXPECT_FALSE(_responses->docvalues); + rapidjson::Document query; + query.Parse(_responses->properties.at("query").c_str()); + ASSERT_FALSE(query.HasParseError()); + ASSERT_EQ(query["_source"].Size(), 1); + EXPECT_STREQ(query["_source"][0].GetString(), "_id"); + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + ASSERT_EQ(rows, 3); + EXPECT_EQ(string_at(output, 0, 2), "c"); +} + +TEST_F(EsV2ReaderTest, MalformedResponseReleasesStream) { + _responses->pages = {"invalid-json"}; + auto scan = reader(); + EXPECT_FALSE(scan->open(request({0})).ok()); + EXPECT_EQ(_responses->closes, 1); +} + +TEST_F(EsV2ReaderTest, FullyFilteredBatchesKeepScanningUntilEof) { + auto scan = reader(); + scan->set_batch_size(1); + auto projection = request({0}); + projection->conjuncts.push_back( + prepared_conjunct(&_state, std::make_shared(0, 10))); + ASSERT_TRUE(scan->open(projection).ok()); + for (int batch = 0; batch < 4; ++batch) { + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + EXPECT_EQ(rows, 0); + EXPECT_EQ(eof, batch == 3); + } + EXPECT_EQ(_io->predicate_filtered_rows, 3); + EXPECT_EQ(_responses->closes, 1); +} + +} // namespace doris::format::es diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java index 539efd7b7ca321..f12830a21c99df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java @@ -47,12 +47,21 @@ import org.apache.doris.thrift.TEsScanNode; import org.apache.doris.thrift.TEsScanRange; import org.apache.doris.thrift.TExplainLevel; +import org.apache.doris.thrift.TExternalScanRange; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TFileScanNode; +import org.apache.doris.thrift.TFileScanRange; +import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TFileScanSlotInfo; +import org.apache.doris.thrift.TFileType; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; import org.apache.doris.thrift.TScanRange; import org.apache.doris.thrift.TScanRangeLocation; import org.apache.doris.thrift.TScanRangeLocations; +import org.apache.doris.thrift.TTableFormatFileDesc; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -78,6 +87,7 @@ public class EsScanNode extends ExternalScanNode { private final EsTablePartitions esTablePartitions; private final EsTable table; + private final boolean useFileScannerV2; private QueryBuilder queryBuilder; private boolean isFinalized = false; @@ -93,6 +103,10 @@ public EsScanNode(PlanNodeId id, TupleDescriptor desc, boolean esExternalTable, table = (EsTable) (desc.getTable()); } esTablePartitions = table.getEsTablePartitions(); + // Planning and range assignment can run on different threads. Capture the query's + // choice once so both always serialize the same scan protocol. + useFileScannerV2 = ConnectContext.get() != null + && ConnectContext.get().getSessionVariable().enableFileScannerV2; } @Override @@ -150,7 +164,18 @@ private int useDocValueScan(TupleDescriptor desc, Map docValueCo @SneakyThrows @Override protected void toThrift(TPlanNode msg) { - msg.node_type = TPlanNodeType.ES_HTTP_SCAN_NODE; + if (useFileScannerV2) { + msg.setNodeType(TPlanNodeType.FILE_SCAN_NODE); + msg.setFileScanNode(new TFileScanNode().setTupleId(desc.getId().asInt()).setTableName(table.getName())); + } else { + msg.setNodeType(TPlanNodeType.ES_HTTP_SCAN_NODE); + msg.setEsScanNode(buildEsScanNode()); + } + super.toThrift(msg); + } + + @SneakyThrows + private TEsScanNode buildEsScanNode() { Map properties = Maps.newHashMap(); if (table.getUserName() != null) { properties.put(EsResource.USER, table.getUserName()); @@ -169,8 +194,65 @@ protected void toThrift(TPlanNode msg) { esScanNode.setFieldsContext(table.fieldsContext()); } esScanNode.setProperties(properties); - msg.es_scan_node = esScanNode; - super.toThrift(msg); + return esScanNode; + } + + @Override + public List getScanRangeLocations(long maxScanRangeLength) { + List shardLocations = super.getScanRangeLocations(maxScanRangeLength); + if (!useFileScannerV2) { + return shardLocations; + } + TEsScanNode esNode = buildEsScanNode(); + TFileScanRangeParams params = new TFileScanRangeParams(); + params.setFormatType(TFileFormatType.FORMAT_ES_HTTP); + params.setFileType(TFileType.FILE_STREAM); + params.setSrcTupleId(-1); + params.setDestTupleId(desc.getId().asInt()); + params.setNumOfColumnsFromFile(desc.getSlots().size()); + params.setRequiredSlots(new ArrayList<>()); + for (SlotDescriptor slot : desc.getSlots()) { + params.addToRequiredSlots(new TFileScanSlotInfo().setSlotId(slot.getId().asInt()).setIsFileSlot(true)); + } + params.setEsProperties(esNode.getProperties()); + if (esNode.isSetDocvalueContext()) { + params.setEsDocvalueContext(esNode.getDocvalueContext()); + } + if (esNode.isSetFieldsContext()) { + params.setEsFieldsContext(esNode.getFieldsContext()); + } + List fileLocations = new ArrayList<>(shardLocations.size()); + for (TScanRangeLocations location : shardLocations) { + TEsScanRange shard = location.getScanRange().getEsScanRange(); + Map shardParams = new HashMap<>(); + shardParams.put("index", shard.getIndex()); + shardParams.put("shard_id", String.valueOf(shard.getShardId())); + if (shard.isSetType()) { + shardParams.put("type", shard.getType()); + } + shardParams.put("es_hosts", shard.getEsHosts().stream() + .map(host -> host.getHostname() + ":" + host.getPort()).collect(Collectors.joining(","))); + TTableFormatFileDesc tableFormat = new TTableFormatFileDesc(); + tableFormat.setTableFormatType("es"); + tableFormat.setEsParams(shardParams); + TFileRangeDesc range = new TFileRangeDesc(); + range.setPath("es://" + shard.getIndex() + "/" + shard.getShardId()); + range.setStartOffset(0); + range.setSize(-1); + range.setFormatType(TFileFormatType.FORMAT_ES_HTTP); + range.setFileType(TFileType.FILE_STREAM); + range.setTableFormatParams(tableFormat); + TFileScanRange fileRange = new TFileScanRange(); + // This branch still uses EsScanNode rather than FileQueryScanNode, so the + // coordinator does not collect its common parameters into file_scan_params. + fileRange.setParams(params); + fileRange.addToRanges(range); + TScanRangeLocations converted = location.deepCopy(); + converted.setScanRange(new TScanRange().setExtScanRange( + new TExternalScanRange().setFileScanRange(fileRange))); + fileLocations.add(converted); + } + return fileLocations; } // only do partition(es index level) prune diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/source/EsScanNodeFileScannerV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/source/EsScanNodeFileScannerV2Test.java new file mode 100644 index 00000000000000..7a109b3184a52a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/source/EsScanNodeFileScannerV2Test.java @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.es.source; + +import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.EsResource; +import org.apache.doris.catalog.EsTable; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.es.QueryBuilders; +import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.planner.ScanContext; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TEsScanRange; +import org.apache.doris.thrift.TFileScanRange; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TPlanNode; +import org.apache.doris.thrift.TPlanNodeType; +import org.apache.doris.thrift.TScanRange; +import org.apache.doris.thrift.TScanRangeLocations; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.List; + +class EsScanNodeFileScannerV2Test { + @AfterEach + void tearDown() { + ConnectContext.remove(); + } + + @Test + void enabledScannerUsesFileScanPlan() throws Exception { + TestScanNode node = createNode(true); + TPlanNode plan = node.serialize(); + Assertions.assertEquals(TPlanNodeType.FILE_SCAN_NODE, plan.getNodeType()); + Assertions.assertEquals(7, plan.getFileScanNode().getTupleId()); + Assertions.assertFalse(plan.isSetEsScanNode()); + } + + @Test + void disabledScannerKeepsLegacyPlanAndRanges() throws Exception { + TestScanNode node = createNode(false); + Assertions.assertEquals(TPlanNodeType.ES_HTTP_SCAN_NODE, node.serialize().getNodeType()); + Assertions.assertTrue(node.getScanRangeLocations(0).get(0).getScanRange().isSetEsScanRange()); + } + + @Test + void enabledScannerConvertsShardToFileRange() throws Exception { + TestScanNode node = createNode(true); + TScanRange range = node.getScanRangeLocations(0).get(0).getScanRange(); + Assertions.assertTrue(range.isSetExtScanRange()); + Assertions.assertFalse(range.isSetEsScanRange()); + Assertions.assertEquals(7, range.getExtScanRange().getFileScanRange().getParams().getDestTupleId()); + Assertions.assertEquals(-1, range.getExtScanRange().getFileScanRange().getParams().getSrcTupleId()); + Assertions.assertEquals("es", range.getExtScanRange().getFileScanRange() + .getRanges().get(0).getTableFormatParams().getTableFormatType()); + TFileScanRange fileRange = range.getExtScanRange().getFileScanRange(); + Assertions.assertEquals(1, fileRange.getParams().getNumOfColumnsFromFile()); + Assertions.assertEquals(4, fileRange.getParams().getRequiredSlots().get(0).getSlotId()); + Assertions.assertTrue(fileRange.getParams().getRequiredSlots().get(0).isIsFileSlot()); + Assertions.assertEquals("{\"match_all\":{}}", fileRange.getParams().getEsProperties().get("query_dsl")); + Assertions.assertEquals("es_index", fileRange.getRanges().get(0) + .getTableFormatParams().getEsParams().get("index")); + Assertions.assertEquals("3", fileRange.getRanges().get(0) + .getTableFormatParams().getEsParams().get("shard_id")); + Assertions.assertEquals("127.0.0.1:9200", fileRange.getRanges().get(0) + .getTableFormatParams().getEsParams().get("es_hosts")); + Assertions.assertEquals(1, node.getScanRangeLocations(0).get(0) + .getScanRange().getExtScanRange().getFileScanRange().getRangesSize()); + } + + @Test + void scanProtocolSurvivesPlanningThreadContext() throws Exception { + TestScanNode node = createNode(true); + ConnectContext.remove(); + Assertions.assertEquals(TPlanNodeType.FILE_SCAN_NODE, node.serialize().getNodeType()); + Assertions.assertTrue(node.getScanRangeLocations(0).get(0).getScanRange().isSetExtScanRange()); + } + + private TestScanNode createNode(boolean enabled) throws Exception { + ConnectContext context = new ConnectContext(); + context.getSessionVariable().enableFileScannerV2 = enabled; + context.setThreadLocalInfo(); + Column column = new Column("id", Type.INT); + EsTable table = new EsTable(1, "es_table", ImmutableList.of(column), ImmutableMap.of( + EsResource.HOSTS, "127.0.0.1:9200", EsResource.INDEX, "es_index", + EsResource.DOC_VALUE_SCAN, "false", EsResource.KEYWORD_SNIFF, "false"), null); + TupleDescriptor tuple = new TupleDescriptor(new TupleId(7)); + tuple.setTable(table); + SlotDescriptor slot = new SlotDescriptor(new SlotId(4), tuple); + slot.setColumn(column); + tuple.addSlot(slot); + TestScanNode node = new TestScanNode(tuple); + Field builder = EsScanNode.class.getDeclaredField("queryBuilder"); + builder.setAccessible(true); + builder.set(node, QueryBuilders.matchAllQuery()); + TEsScanRange esRange = new TEsScanRange(); + esRange.setIndex("es_index"); + esRange.setShardId(3); + esRange.setEsHosts(ImmutableList.of(new TNetworkAddress("127.0.0.1", 9200))); + TScanRangeLocations locations = new TScanRangeLocations(); + locations.setScanRange(new TScanRange().setEsScanRange(esRange)); + node.installRanges(ImmutableList.of(locations)); + return node; + } + + private static class TestScanNode extends EsScanNode { + TestScanNode(TupleDescriptor tuple) { + super(new PlanNodeId(1), tuple, false, ScanContext.EMPTY); + } + + TPlanNode serialize() { + TPlanNode plan = new TPlanNode(); + toThrift(plan); + return plan; + } + + void installRanges(List ranges) { + scanRangeLocations = ranges; + } + } +} diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 5a9a55cabe56ef..9d9a042c469c1c 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -633,6 +633,8 @@ struct TTableFormatFileDesc { // JDBC connection parameters (used when table_format_type == "jdbc") 11: optional map jdbc_params 12: optional TLanceFileDesc lance_params + // Keep ES field ids aligned with master for mixed-version requests. + 13: optional map es_params } // Deprecated, hive text talbe is a special format, not a serde type @@ -709,6 +711,9 @@ struct TFileScanRangeParams { // Paimon options from FE, used for jni/native scanner // Set at ScanNode level to avoid redundant serialization in each split 30: optional map paimon_options + 31: optional map es_properties + 32: optional map es_docvalue_context + 33: optional map es_fields_context // Versioned Iceberg scan semantics negotiated by FE. Absence/zero preserves legacy BE // behavior during a BE-first rolling upgrade; version 1 enables file-wide ID projection and // logical initial-default materialization. @@ -718,7 +723,6 @@ struct TFileScanRangeParams { // HMS catalog property hive.parquet.time-zone. Interpretation is versioned by // parquet_timestamp_semantics_version. 36: optional string hive_parquet_time_zone - // 31-33 are used in master; do not allocate them in branch-4.1. 37: optional TLanceScanParams lance_scan_params // Non-regular columns in the pinned full schema, including columns pruned from phase one. // When present, omitted names are REGULAR. Used to rebuild row-id fetch projections. diff --git a/regression-test/suites/external_table_p2/es/test_es_file_scanner_v2.groovy b/regression-test/suites/external_table_p2/es/test_es_file_scanner_v2.groovy new file mode 100644 index 00000000000000..e23dce31b8577d --- /dev/null +++ b/regression-test/suites/external_table_p2/es/test_es_file_scanner_v2.groovy @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_es_file_scanner_v2", "p2,external,es,external_docker,external_docker_es") { + String enabled = context.config.otherConfigs.get("enableEsTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + def originalBatchSize = sql("show variables like 'batch_size'")[0][1] + try { + // Small batches force scroll pagination and repeated output block production. + sql "set batch_size = 1" + [5, 6, 7, 8].each { version -> + String port = context.config.otherConfigs.get("es_${version}_port") + String timeIndex = version <= 6 ? "test2_20220808" : "test2" + [false, true].each { docvalues -> + String catalog = "test_es_v2_${version}_${docvalues}" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties ( + "type" = "es", + "hosts" = "http://${externalEnvIp}:${port}", + "nodes_discovery" = "false", + "enable_keyword_sniff" = "true", + "mapping_es_id" = "true", + "enable_docvalue_scan" = "${docvalues}" + )""" + sql "switch ${catalog}" + def queries = [ + "select test2, test1, test6 from test1 order by test1, test2, test6", + "select _id, test1 from test1 order by _id", + "select c_bool, c_integer, c_keyword, c_person from test1 order by test1", + "select test1 from ${timeIndex} where substring(test2, 2) = 'ext2' order by test1", + "select test1 from test1 where test1 like 'string%' order by test1", + "select count(*), count(message) from test1", + "select test1 from test1 where message is null order by test1", + "select test1 from test1 where test1 = 'missing-value'", + "select test1 from test1 order by test1 limit 2", + "select count(*) from (select * from test1 limit 2) t", + "select * from composite_type_array order by name" + ] + // Compare against the existing scanner on the same index and ES version instead + // of embedding fixture results that can vary across the external Docker images. + queries.each { query -> + sql "set enable_file_scanner_v2 = false" + def expected = sql(query) + sql "set enable_file_scanner_v2 = true" + def actual = sql(query) + assertEquals(expected, actual, "ES ${version}, docvalues=${docvalues}: ${query}") + } + } + } + } finally { + sql "set enable_file_scanner_v2 = ${originalV2}" + sql "set batch_size = ${originalBatchSize}" + } +}