Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions be/src/exprs/function/function_search.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,31 @@ Status FunctionSearch::evaluate_inverted_index_with_search_param(
const IndexExecContext* index_exec_ctx,
const std::unordered_map<std::string, int>& field_name_to_column_id,
const std::shared_ptr<IndexQueryContext>& index_query_context) const {
// VSearchExpr enters here directly, outside IFunction::execute() and its exception boundary,
// so this is where a failure inside the search, such as an analyzer whose first token
// stream throws, has to become a Status.
try {
return evaluate_inverted_index_with_search_param_unguarded(
search_param, data_type_with_names, std::move(iterators), num_rows, bitmap_result,
enable_cache, index_exec_ctx, field_name_to_column_id, index_query_context);
} catch (const CLuceneError& e) {
return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("search failed: {}",
e.what());
} catch (const Exception& e) {
return e.to_status();
}
}

// The search body stays together so the public entry point can convert thrown exceptions.
// NOLINTNEXTLINE(readability-function-cognitive-complexity,readability-function-size)
Status FunctionSearch::evaluate_inverted_index_with_search_param_unguarded(
const TSearchParam& search_param,
const std::unordered_map<std::string, IndexFieldNameAndTypePair>& data_type_with_names,
std::unordered_map<std::string, IndexIterator*> iterators, uint32_t num_rows,
InvertedIndexResultBitmap& bitmap_result, bool enable_cache,
const IndexExecContext* index_exec_ctx,
const std::unordered_map<std::string, int>& field_name_to_column_id,
const std::shared_ptr<IndexQueryContext>& index_query_context) const {
const bool is_nested_query = search_param.root.clause_type == "NESTED";
if (is_nested_query && !is_nested_group_search_supported()) {
return Status::NotSupported(
Expand Down
10 changes: 10 additions & 0 deletions be/src/exprs/function/function_search.h
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,16 @@ class FunctionSearch : public IFunction {
const std::unordered_map<std::string, int>& field_name_to_column_id,
const std::shared_ptr<IndexQueryContext>& index_query_context = nullptr) const;

// The body of the overload above; it may throw, so only that overload calls it.
Status evaluate_inverted_index_with_search_param_unguarded(
const TSearchParam& search_param,
const std::unordered_map<std::string, IndexFieldNameAndTypePair>& data_type_with_names,
std::unordered_map<std::string, IndexIterator*> iterators, uint32_t num_rows,
InvertedIndexResultBitmap& bitmap_result, bool enable_cache,
const IndexExecContext* index_exec_ctx,
const std::unordered_map<std::string, int>& field_name_to_column_id,
const std::shared_ptr<IndexQueryContext>& index_query_context) const;

// Public methods for testing
enum class ClauseTypeCategory {
NON_TOKENIZED, // TERM, PREFIX, WILDCARD, REGEXP, RANGE, LIST - no tokenization, use EQUAL_QUERY
Expand Down
24 changes: 7 additions & 17 deletions be/src/exprs/function/match.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,7 @@ Status FunctionMatchBase::evaluate_inverted_index(
}
const std::string& function_name = get_name();

if (function_name == MATCH_PHRASE_FUNCTION || function_name == MATCH_PHRASE_PREFIX_FUNCTION ||
function_name == MATCH_PHRASE_EDGE_FUNCTION) {
auto reader = iter->get_reader(InvertedIndexReaderType::FULLTEXT);
if (reader && !segment_v2::IndexReaderHelper::is_support_phrase(reader)) {
return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>(
"phrase queries require setting support_phrase = true");
}
}
// support_phrase is checked once the analyzer has selected the reader that runs the query.
Field param_value;
arguments[0].column->get(0, param_value);
if (param_value.is_null()) {
Expand Down Expand Up @@ -199,10 +192,8 @@ std::vector<segment_v2::TermInfo> FunctionMatchBase::analyse_query_str_token(
VLOG_DEBUG << "begin to run " << get_name() << ", parser_type: "
<< inverted_index_parser_type_to_string(analyzer_ctx->parser_type);

// Decision is based on parser_type (from index properties):
// - PARSER_NONE: no tokenization (keyword/exact match)
// - Other parsers: tokenize using the analyzer
if (!analyzer_ctx->should_tokenize()) {
// A named analyzer also requires analysis when the parser is none.
if (!analyzer_ctx->requires_analysis()) {
// Keyword index: all strings (including empty) are valid tokens for exact match.
// Empty string is a valid value in keyword index and should be matchable.
query_tokens.emplace_back(match_query_str);
Expand Down Expand Up @@ -235,15 +226,14 @@ inline std::vector<segment_v2::TermInfo> FunctionMatchBase::analyse_data_token(
return data_tokens;
}

// Determine tokenization strategy based on parser_type
const bool should_tokenize =
analyzer_ctx->should_tokenize() && analyzer_ctx->analyzer != nullptr;
const bool requires_analysis =
analyzer_ctx->requires_analysis() && analyzer_ctx->analyzer != nullptr;

if (array_offsets) {
for (auto next_src_array_offset = (*array_offsets)[current_block_row_idx];
current_src_array_offset < next_src_array_offset; ++current_src_array_offset) {
const auto& str_ref = string_col->get_data_at(current_src_array_offset);
if (!should_tokenize) {
if (!requires_analysis) {
data_tokens.emplace_back(str_ref.to_string());
continue;
}
Expand All @@ -256,7 +246,7 @@ inline std::vector<segment_v2::TermInfo> FunctionMatchBase::analyse_data_token(
}
} else {
const auto& str_ref = string_col->get_data_at(current_block_row_idx);
if (!should_tokenize) {
if (!requires_analysis) {
data_tokens.emplace_back(str_ref.to_string());
} else {
auto reader = doris::segment_v2::inverted_index::InvertedIndexAnalyzer::create_reader(
Expand Down
37 changes: 37 additions & 0 deletions be/src/exprs/function/variant_inverted_index_search.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,45 @@ void add_search_binding_diagnostic(const std::shared_ptr<IndexQueryContext>& con
}
}

InvertedIndexAnalyzerCtxSPtr build_analyzer_context_unsafe(
const std::map<std::string, std::string>& properties, const std::string& analyzer_key) {
InvertedIndexAnalyzerConfig config;
config.analyzer_name = get_analyzer_name_from_properties(properties);
config.parser_type = get_inverted_index_parser_type_from_string(
get_parser_string_from_properties(properties));
config.parser_mode = get_parser_mode_string_from_properties(properties);
config.lower_case = get_parser_lowercase_from_properties(properties);
config.stop_words = get_parser_stopwords_from_properties(properties);
config.char_filter_map = get_parser_char_filter_map_from_properties(properties);

auto analyzer_context = std::make_shared<InvertedIndexAnalyzerCtx>();
analyzer_context->analyzer_key = analyzer_key;
analyzer_context->analyzer_name = config.analyzer_name;
analyzer_context->parser_type = config.parser_type;
analyzer_context->char_filter_map = config.char_filter_map;
if (analyzer_context->requires_analysis()) {
analyzer_context->analyzer_provider =
inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(&config);
}
return analyzer_context;
}

} // namespace

// Replayed components can collide across policy families, so building the provider throws.
Result<InvertedIndexAnalyzerCtxSPtr> build_search_analyzer_context(
const std::map<std::string, std::string>& properties, const std::string& analyzer_key) {
try {
return build_analyzer_context_unsafe(properties, analyzer_key);
} catch (const CLuceneError& error) {
return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>(
"Build search analyzer failed: {}", error.what()));
} catch (const Exception& error) {
return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>(
"Build search analyzer failed: {}", error.what()));
}
}

FieldReaderResolver::FieldReaderResolver(
const std::unordered_map<std::string, IndexFieldNameAndTypePair>& data_type_with_names,
const std::unordered_map<std::string, IndexIterator*>& iterators,
Expand Down
5 changes: 5 additions & 0 deletions be/src/exprs/function/variant_inverted_index_search.h
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@ struct FieldReaderBinding {
}
};

// Build the analyzer context a SEARCH binding executes with, converting a failure to build the
// analyzer provider into a Status instead of letting the exception escape a Status-returning caller.
Result<InvertedIndexAnalyzerCtxSPtr> build_search_analyzer_context(
const std::map<std::string, std::string>& properties, const std::string& analyzer_key);

class FieldReaderResolver {
public:
FieldReaderResolver(
Expand Down
53 changes: 39 additions & 14 deletions be/src/exprs/vmatch_predicate.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,14 @@ namespace doris {
using namespace doris::segment_v2;

VMatchPredicate::VMatchPredicate(const TExprNode& node) : VExpr(node) {
// Step 1: Create configuration (stack-allocated temporary, follows SRP)
const auto resolved = AnalyzerConfigParser::parse(
node.match_predicate.analyzer_name, node.match_predicate.parser_type,
node.match_predicate.parser_mode, node.match_predicate.parser_lowercase,
node.match_predicate.char_filter_map);

InvertedIndexAnalyzerConfig config;
config.analyzer_name = node.match_predicate.analyzer_name;
config.parser_type =
get_inverted_index_parser_type_from_string(node.match_predicate.parser_type);
config.analyzer_name = resolved.provider_name;
config.parser_type = resolved.parser_type;
config.parser_mode = node.match_predicate.parser_mode;
config.char_filter_map = node.match_predicate.char_filter_map;
if (node.match_predicate.parser_lowercase) {
Expand All @@ -74,18 +77,40 @@ VMatchPredicate::VMatchPredicate(const TExprNode& node) : VExpr(node) {
{ config.lower_case = ""; })
config.stop_words = node.match_predicate.parser_stopwords;

// Step 2: Use config to create analyzer (factory method).
// Always create analyzer based on parser_type for slow path (tables without index).
// For index path, FullTextIndexReader will check analyzer_name to decide whether
// to use this analyzer or fallback to index's own analyzer.
_analyzer = inverted_index::InvertedIndexAnalyzer::create_analyzer(&config);

// Step 3: Create runtime context (only extract runtime-needed info)
_analyzer_ctx = std::make_shared<InvertedIndexAnalyzerCtx>();
_analyzer_ctx->analyzer_name = config.analyzer_name;
_analyzer_ctx->parser_type = config.parser_type;
_analyzer_ctx->analyzer_key = resolved.analyzer_key;
_analyzer_ctx->analyzer_name = resolved.provider_name;
_analyzer_ctx->parser_type = resolved.parser_type;

if (_analyzer_ctx->requires_analysis()) {
std::string bound_name;
std::string legacy_name;
_analyzer_provider = inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(
&config, &bound_name, &legacy_name);
_analyzer = _analyzer_provider->get_analyzer();
if (!legacy_name.empty() && legacy_name != bound_name) {
_analyzer_ctx->legacy_analyzer_key =
AnalyzerConfigParser::parse(legacy_name, node.match_predicate.parser_type,
node.match_predicate.parser_mode,
node.match_predicate.parser_lowercase,
node.match_predicate.char_filter_map)
.analyzer_key;
}
if (bound_name != resolved.provider_name) {
// Reader selection and query tokenization must use the same policy binding.
_analyzer_ctx->analyzer_key =
AnalyzerConfigParser::parse(bound_name, node.match_predicate.parser_type,
node.match_predicate.parser_mode,
node.match_predicate.parser_lowercase,
node.match_predicate.char_filter_map)
.analyzer_key;
_analyzer_ctx->analyzer_name = std::move(bound_name);
}
}

_analyzer_ctx->char_filter_map = std::move(config.char_filter_map);
_analyzer_ctx->analyzer = _analyzer;
_analyzer_ctx->analyzer_provider = _analyzer_provider;
}

VMatchPredicate::~VMatchPredicate() = default;
Expand Down Expand Up @@ -153,7 +178,7 @@ Status VMatchPredicate::evaluate_inverted_index(VExprContext* context, uint32_t
}

const std::string& VMatchPredicate::get_analyzer_key() const {
return _analyzer_ctx->analyzer_name;
return _analyzer_ctx->analyzer_key;
}

Status VMatchPredicate::execute_column(VExprContext* context, const Block* block,
Expand Down
4 changes: 4 additions & 0 deletions be/src/exprs/vmatch_predicate.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ class VMatchPredicate final : public VExpr {
const std::string& expr_name() const override;
const std::string& function_name() const;
[[nodiscard]] const std::string& get_analyzer_key() const override;
[[nodiscard]] const InvertedIndexAnalyzerCtx* query_analyzer_ctx() const {
return _analyzer_ctx.get();
}

std::string debug_string() const override;

Expand All @@ -69,6 +72,7 @@ class VMatchPredicate final : public VExpr {

// Lifecycle management: holds ownership of the analyzer
std::shared_ptr<lucene::analysis::Analyzer> _analyzer;
segment_v2::inverted_index::AnalyzerProviderPtr _analyzer_provider;

// Runtime context: holds raw pointer to analyzer and necessary runtime info
InvertedIndexAnalyzerCtxSPtr _analyzer_ctx;
Expand Down
Loading
Loading