From bdde929a6c1f3dc9a18fed8da3e90429afc40da3 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Thu, 29 Oct 2026 06:02:09 +0000 Subject: [PATCH] [fix](be) Score regexp and wildcard patterns with a constant like Elasticsearch ### What problem does this PR solve? Issue Number: None Related PR: #69052 Problem Summary: score() of pattern queries disagrees with Elasticsearch and with itself: -- 'Inverted indexes are suitable for full text search', -- 'The parser splits text into multiple terms', ... SELECT score() FROM document WHERE SEARCH('content:/.*ext/'); -- BM25 sum SELECT score() FROM document WHERE SEARCH('content:*ext'); -- BM25 sum SELECT score() FROM document WHERE SEARCH('content:te*'); -- 1 SELECT score() FROM document WHERE content MATCH_REGEXP 'ext'; -- 0 Elasticsearch scores a query_string /regexp/, wildcard or prefix, and a regexp, wildcard or prefix query, with a constant 1 per matching row (default rewrite constant_score_blended); measured on 9.5.5 with the rows above: 1.0 for both matching rows. Only an explicit "rewrite": "scoring_boolean" sums the BM25 of the expanded terms. Next to a scored clause in a bool must, a regexp adds exactly 1. #69052 made SEARCH regexp and wildcard sum the BM25 of every expanded term, i.e. scoring_boolean always on, while prefix stayed constant. That sum also takes idf and avgdl from the current segment, not from the tablet statistics MATCH uses: two inserts of the same text scored 0.2876821 and 1.3862942 for identical 'apple' rows, while MATCH 'apple' scored 0.6931471 for both. MATCH_REGEXP never requested a scoring context, so it scored 0 like a filter. Fix: 1. Revert the scoring part of #69052: ExpandWeight scores every pattern with the constant scorer again. The test fixture extensions that #69051 built on are kept. 2. MATCH_REGEXP requests scoring on every index; the regexp expansion then publishes the constant 1. The MATCH statistics collector skips it, since a constant reads no statistics and the pattern is not text to analyze. ### Release note score() of SEARCH regexp and wildcard clauses is a constant 1 per matching row again, like prefix clauses and Elasticsearch's default. score() of MATCH_REGEXP is 1 instead of 0. ### Check List (For Author) - Test: Regression test / Unit Test / Manual test - Regression test: new search/test_search_score_pattern_constant (V2/V3/SNII, two segments, keyword and analyzed fields; identical on all three formats); 24 existing score suites in inverted_index_p0, search and variant_p0 pass unchanged - Unit Test: 477 tests (IndexReaderHelperTest, CollectionStatistics*, regexp/wildcard/prefix engine tests restored by the revert, FunctionSearchTest, SNII query tests). Mutation checks: the gate test fails without the gate change, the collector test fails without the regexp skip - Manual test: Elasticsearch 9.5.5 on the same rows for every query shape above; two-segment table now scores 1 for every pattern hit - Built and run on master 62e3d81a42 plus this change: the current master tip does not compile (function_datasketches_hll.cpp after #69679), and none of the files touched here changed in between - Behavior changed: Yes. SEARCH regexp/wildcard scores and MATCH_REGEXP scores change as described in the release note. - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 --- be/src/storage/index/index_reader_helper.cpp | 10 ++ .../query/engine/expand_query/expand_query.h | 5 +- .../engine/expand_query/expand_weight.cpp | 69 +--------- .../query/engine/expand_query/expand_weight.h | 9 +- .../inverted/reader/predicate_collector.cpp | 5 + .../function/search/function_search_test.cpp | 23 ++-- .../query/engine/prefix_query_test.cpp | 23 ---- .../query/engine/regexp_query_test.cpp | 48 ++----- .../query/engine/wildcard_query_test.cpp | 54 ++------ .../reader/collection_statistics_test.cpp | 46 +++++++ .../segment/index_reader_helper_test.cpp | 26 ++++ .../test_search_score_pattern_constant.out | 124 ++++++++++++++++++ .../test_search_score_pattern_constant.groovy | 80 +++++++++++ 13 files changed, 325 insertions(+), 197 deletions(-) create mode 100644 regression-test/data/search/test_search_score_pattern_constant.out create mode 100644 regression-test/suites/search/test_search_score_pattern_constant.groovy diff --git a/be/src/storage/index/index_reader_helper.cpp b/be/src/storage/index/index_reader_helper.cpp index 5607169cd08d77..51074a7dca3696 100644 --- a/be/src/storage/index/index_reader_helper.cpp +++ b/be/src/storage/index/index_reader_helper.cpp @@ -36,6 +36,12 @@ bool can_score_match(const TabletIndex* index_meta) { bool IndexReaderHelper::is_need_similarity_score(InvertedIndexQueryType query_type, const TabletIndex* index_meta) { + // A regexp scores a constant 1 per matching row, as an Elasticsearch regexp query does in + // query context. The constant reads no statistics, norms or positions, so every index can + // score it: body MATCH 'apple' AND body MATCH_REGEXP 'ap.*' adds 1 to the apple score. + if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) { + return true; + } return (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY || query_type == InvertedIndexQueryType::MATCH_ALL_QUERY || query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY || @@ -45,6 +51,10 @@ bool IndexReaderHelper::is_need_similarity_score(InvertedIndexQueryType query_ty bool IndexReaderHelper::is_need_similarity_score(TExprOpcode::type query_type, const TabletIndex* index_meta) { + // See the InvertedIndexQueryType overload: a regexp scores a constant on every index. + if (query_type == TExprOpcode::MATCH_REGEXP) { + return true; + } return (query_type == TExprOpcode::MATCH_ANY || query_type == TExprOpcode::MATCH_ALL || query_type == TExprOpcode::MATCH_PHRASE || query_type == TExprOpcode::MATCH_PHRASE_PREFIX) && diff --git a/be/src/storage/index/inverted/query/engine/expand_query/expand_query.h b/be/src/storage/index/inverted/query/engine/expand_query/expand_query.h index 155acf16c73021..45816618fc9e40 100644 --- a/be/src/storage/index/inverted/query/engine/expand_query/expand_query.h +++ b/be/src/storage/index/inverted/query/engine/expand_query/expand_query.h @@ -38,8 +38,9 @@ class ExpandQuery : public Query { _pattern(std::move(pattern)) {} ~ExpandQuery() override = default; - WeightPtr weight(bool enable_scoring) override { - return std::make_shared(_context, _field, _kind, _pattern, enable_scoring); + // Scores are constant, so scoring changes nothing. + WeightPtr weight(bool /*enable_scoring*/) override { + return std::make_shared(_context, _field, _kind, _pattern); } private: diff --git a/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.cpp b/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.cpp index 85fd10a51d0ef4..3c55f171ec4786 100644 --- a/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.cpp +++ b/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.cpp @@ -17,7 +17,6 @@ #include "storage/index/inverted/query/engine/expand_query/expand_weight.h" -#include #include #include #include @@ -26,14 +25,10 @@ #include "storage/index/inverted/query/engine/bit_set_query/bit_set_scorer.h" #include "storage/index/inverted/query/engine/const_score_query/const_score_scorer.h" #include "storage/index/inverted/query/engine/nullable_scorer.h" -#include "storage/index/inverted/query/engine/score_combiner.h" -#include "storage/index/inverted/query/engine/term_query/term_scorer.h" -#include "storage/index/inverted/query/engine/union/buffered_union.h" #include "storage/index/inverted/query/exec/block_doc_set.h" #include "storage/index/inverted/query/exec/collect_postings.h" #include "storage/index/inverted/query/exec/roaring_docid_sink.h" #include "storage/index/inverted/query/exec/term_waves.h" -#include "storage/index/inverted/query/scoring/bm25_similarity.h" #include "storage/index/inverted/query/spi/postings_cursor.h" namespace doris::segment_v2::inverted_index::query_v2 { @@ -63,15 +58,11 @@ Status collect_expanded_rows(index_query::IndexSource& source, index_query::Term } ExpandWeight::ExpandWeight(IndexQueryContextPtr context, std::wstring field, - index_query::TermPatternKind kind, std::string pattern, - bool enable_scoring) + index_query::TermPatternKind kind, std::string pattern) : _context(std::move(context)), _field(std::move(field)), _kind(kind), - _pattern(std::move(pattern)), - // Prefix and phrase-edge expansions retain their constant-score behavior. - _enable_scoring(enable_scoring && (kind == index_query::TermPatternKind::kRegexp || - kind == index_query::TermPatternKind::kWildcard)) {} + _pattern(std::move(pattern)) {} std::shared_ptr ExpandWeight::_rows(const QueryExecutionContext& context, const std::string& binding_key, @@ -94,68 +85,12 @@ std::shared_ptr ExpandWeight::_rows(const QueryExecutionContex ScorerPtr ExpandWeight::scorer(const QueryExecutionContext& context, const std::string& binding_key) { - // The listing fast path discards term frequencies, so scored patterns need term scorers. - if (_enable_scoring) { - return make_nullable_scorer(_scored_terms(context, binding_key), - logical_field_or_fallback(context, binding_key, _field), - context.null_resolver); - } auto scorer = std::make_shared>( std::make_shared(_rows(context, binding_key, nullptr))); return make_nullable_scorer(scorer, logical_field_or_fallback(context, binding_key, _field), context.null_resolver); } -ScorerPtr ExpandWeight::_scored_terms(const QueryExecutionContext& context, - const std::string& binding_key) { - index_query::TermPattern pattern; - THROW_IF_ERROR(index_query::TermPattern::create(_kind, _pattern, &pattern)); - auto source = lookup_source(_field, context, binding_key); - // An unbound field has no postings and contributes no matches. - if (source == nullptr) { - return std::make_shared(); - } - std::vector terms; - THROW_IF_ERROR(source->expand_terms( - pattern, index_query::expansion_limit(_kind, index_query::max_expansions(*_context)), - &terms)); - // A valid pattern may match no dictionary terms. - if (terms.empty()) { - return std::make_shared(); - } - // Expansion terms are absent from collection statistics; use the bound source's statistics. - index_query::IndexScoringStats stats; - THROW_IF_ERROR(source->scoring_stats(&stats)); - DORIS_CHECK_GT(stats.doc_count, 0); - // Docs-only SNII still reports its physical token count. Only legacy sources without - // that metadata report zero, so use unit average length only for those sources. - const uint64_t token_count = stats.token_count != 0 ? stats.token_count : stats.doc_count; - const float avgdl = static_cast(token_count) / stats.doc_count; - DORIS_CHECK_GT(avgdl, 0.0F); - std::vector scorers; - scorers.reserve(terms.size()); - auto logical_field = logical_field_or_fallback(context, binding_key, _field); - for (const auto& term : terms) { - uint64_t doc_freq = 0; - THROW_IF_ERROR(source->doc_freq(term, &doc_freq)); - // The immutable source just enumerated this term and must have its postings. - DCHECK_GT(doc_freq, 0); - const auto idf = static_cast(std::log1p( - (static_cast(stats.doc_count) - doc_freq + 0.5) / (doc_freq + 0.5))); - auto similarity = std::make_shared(idf, avgdl); - auto postings = open_postings(*source, term, /*positions=*/false, /*enable_scoring=*/true, - similarity); - DCHECK(postings != nullptr); - scorers.emplace_back(std::make_shared(std::move(postings), - std::move(similarity), logical_field)); - } - // One expanded term needs no union; multiple terms add their per-document BM25 scores. - if (scorers.size() == 1) { - return std::move(scorers.front()); - } - return make_buffered_union(scorers, std::make_shared()); -} - index_query::TruthSet ExpandWeight::listed_rows(const QueryExecutionContext& context, const std::string& binding_key, const roaring::Roaring* candidates) { diff --git a/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.h b/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.h index 3bbe43531c311c..b5e979aca5d910 100644 --- a/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.h +++ b/be/src/storage/index/inverted/query/engine/expand_query/expand_weight.h @@ -29,25 +29,23 @@ Status collect_expanded_rows(index_query::IndexSource& source, index_query::Term int32_t max_expansions, const roaring::Roaring* candidates, roaring::Roaring* rows); -// Every document that holds an expanded term; scored regexp and wildcard clauses sum BM25. +// Every document that holds a term the pattern expands to, with a constant score. class ExpandWeight : public Weight { public: ExpandWeight(IndexQueryContextPtr context, std::wstring field, - index_query::TermPatternKind kind, std::string pattern, - bool enable_scoring = false); + index_query::TermPatternKind kind, std::string pattern); ~ExpandWeight() override = default; ScorerPtr scorer(const QueryExecutionContext& context, const std::string& binding_key) override; bool lists_rows(const QueryExecutionContext&, const std::string&) const override { - return !_enable_scoring; + return true; } index_query::TruthSet listed_rows(const QueryExecutionContext& context, const std::string& binding_key, const roaring::Roaring* candidates) override; private: - ScorerPtr _scored_terms(const QueryExecutionContext& context, const std::string& binding_key); std::shared_ptr _rows(const QueryExecutionContext& context, const std::string& binding_key, const roaring::Roaring* candidates); @@ -55,7 +53,6 @@ class ExpandWeight : public Weight { std::wstring _field; index_query::TermPatternKind _kind; std::string _pattern; - bool _enable_scoring; }; } // namespace doris::segment_v2::inverted_index::query_v2 diff --git a/be/src/storage/index/inverted/reader/predicate_collector.cpp b/be/src/storage/index/inverted/reader/predicate_collector.cpp index 6f1fab50075ae8..53c0fed022e386 100644 --- a/be/src/storage/index/inverted/reader/predicate_collector.cpp +++ b/be/src/storage/index/inverted/reader/predicate_collector.cpp @@ -369,6 +369,11 @@ Status MatchPredicateCollector::collect(RuntimeState* state, const TabletSchemaS !IndexReaderHelper::is_need_similarity_score(expr->op(), index_meta)) { return Status::OK(); } + // A regexp scores a constant and reads no statistics. Its value is a pattern, not text, so + // analyzing it would collect statistics for made-up terms: 'ap.*' would become "ap". + if (expr->op() == TExprOpcode::MATCH_REGEXP) { + return Status::OK(); + } DORIS_CHECK(analyzer_ctx->analyzer_provider != nullptr); auto options = DataTypeSerDe::get_default_format_options(); diff --git a/be/test/exprs/function/search/function_search_test.cpp b/be/test/exprs/function/search/function_search_test.cpp index ee9cb7648851ea..cec322d67ef2d4 100644 --- a/be/test/exprs/function/search/function_search_test.cpp +++ b/be/test/exprs/function/search/function_search_test.cpp @@ -40,7 +40,6 @@ #include #include -#include #include #include #include @@ -1881,8 +1880,6 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryExecutesSelectedSniiWildcardReader) std::make_shared(&decoy_meta, decoy_file_reader); auto selected_reader = std::make_shared( &selected_meta, selected_file_reader); - selected_reader->fake_source->set_doc_count(4); - selected_reader->fake_source->set_scoring_stats({.doc_count = 4, .token_count = 3}); selected_reader->fake_source->add("alpha", {0, 2}); selected_reader->fake_source->add("gamma", {1}); decoy_reader->fake_source->add("alpha", {1}); @@ -1931,8 +1928,7 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryExecutesSelectedSniiWildcardReader) binding_key); ASSERT_NE(nullptr, scorer); EXPECT_EQ(0U, scorer->doc()); - // The source has three tokens over four rows, even without stored norms. - EXPECT_NEAR(0.88F * std::log(2.0F), scorer->score(), 1e-6F); + EXPECT_FLOAT_EQ(1.0F, scorer->score()); expect_bitmap_eq(collect_docs(scorer), {0, 2}); // The pattern was lowercased by the selected analyzer and enumerated on the selected // index only; a leading wildcard has no prefix to enumerate from. @@ -2526,8 +2522,7 @@ static void search_snii_pattern(const std::map& proper ASSERT_TRUE(status.ok()) << status.to_string(); EXPECT_EQ(0, reader->query_calls) << value; // The scorer enumerates the stem on the source; the reader itself is never asked. - // This helper checks normalization and routing, without requesting scoring statistics. - auto weight = query->weight(false); + auto weight = query->weight(true); ASSERT_NE(nullptr, weight); auto scorer = weight->scorer(build_variant_search_query_execution_context(4, resolver, nullptr), binding_key); @@ -2760,12 +2755,13 @@ TEST_F(FunctionSearchTest, TestSniiScoredLeafRanksByTheEngineBm25) { } } -// Scored wildcard leaves read frequencies and statistics through the unified source, -// even when the old reader query API publishes no per-document scores. -TEST_F(FunctionSearchTest, TestSniiNativeWildcardUsesSourceScoringStatistics) { +// A leaf for which the reader publishes no per-document score must keep the constant that the +// CLucene path also gives its own constant-score leaves, so that routing scored clauses through a +// new scorer does not quietly re-rank the unscored ones. The fake reader is what withholds the +// scores here, so this pins the "nothing published -> BitSetQuery -> 1.0" behaviour; it does not +// pin which query types the production is_need_similarity_score gate rejects. +TEST_F(FunctionSearchTest, TestSniiNativeLeafWithoutPublishedScoresKeepsConstantScore) { SniiScoringFixture fixture(43, 4); - fixture.reader->fake_source->set_doc_count(4); - fixture.reader->fake_source->set_scoring_stats({.doc_count = 4, .token_count = 2}); fixture.reader->fake_source->add("alpha", {0, 2}); inverted_index::query_v2::QueryPtr query; @@ -2782,8 +2778,7 @@ TEST_F(FunctionSearchTest, TestSniiNativeWildcardUsesSourceScoringStatistics) { auto scorer = weight->scorer(exec_ctx, binding_key); ASSERT_NE(nullptr, scorer); EXPECT_EQ(0U, scorer->doc()); - // avgdl = 2/4 and implicit length = 1: BM25 is idf * 2.2 / 3.1. - EXPECT_NEAR((2.2F / 3.1F) * std::log(2.0F), scorer->score(), 1e-6F); + EXPECT_FLOAT_EQ(1.0F, scorer->score()); expect_bitmap_eq(collect_docs(scorer), {0, 2}); } diff --git a/be/test/storage/index/inverted/query/engine/prefix_query_test.cpp b/be/test/storage/index/inverted/query/engine/prefix_query_test.cpp index e711deb3ed9b23..4d68efdf47adce 100644 --- a/be/test/storage/index/inverted/query/engine/prefix_query_test.cpp +++ b/be/test/storage/index/inverted/query/engine/prefix_query_test.cpp @@ -18,7 +18,6 @@ #include #include -#include #include #include #include @@ -33,8 +32,6 @@ #include "storage/index/inverted/inverted_index_reader.h" #include "storage/index/inverted/query/engine/expand_query/expand_query.h" #include "storage/index/inverted/query/logical/node.h" -#include "storage/index/inverted/query/scoring/bm25_similarity.h" -#include "storage/index/inverted/query/scoring/collection_similarity.h" #include "storage/index/inverted/query/spi/fake_index_source.h" #include "storage/olap_common.h" @@ -495,8 +492,6 @@ TEST(ExpansionLeafTest, KeepsPatternLimitsAndLeavesRootCandidatesToTheCaller) { auto source = std::make_shared(); source->batches = batches; source->set_doc_count(10); - // Scored regexp and wildcard leaves need statistics from their bound source. - source->set_scoring_stats({.doc_count = 10, .token_count = 10}); source->add("apple", {0, 4}); source->add("application", {1}); source->add("apply", {2, 4}); @@ -514,30 +509,12 @@ TEST(ExpansionLeafTest, KeepsPatternLimitsAndLeavesRootCandidatesToTheCaller) { expected.add(2); } for (const bool scoring : {false, true}) { - context->collection_similarity = - scoring ? std::make_shared() : nullptr; auto result = std::make_shared(); result->add(9); const auto status = run_leaf(context, L"content", leaf, &candidates, scoring, source, source->doc_count(), result); ASSERT_TRUE(status.ok()) << status; EXPECT_EQ(*result, expected) << pattern << " scoring=" << scoring; - if (scoring) { - const auto& scores = context->collection_similarity->_bm25_scores; - EXPECT_EQ(scores.size(), expected.cardinality() - 1); - const bool bm25 = kind == logical::ExpandKind::kWildcard || - kind == logical::ExpandKind::kRegexp; - for (const auto& [doc, score] : scores) { - // Row 9 predates this leaf and must not acquire a score from it. - EXPECT_NE(doc, 9U); - EXPECT_TRUE(expected.contains(doc)); - const float df = doc == 1 ? 1.0F : 2.0F; - const float idf = std::log1p((10.0F - df + 0.5F) / (df + 0.5F)); - const float expected_score = - bm25 ? BM25Similarity(idf, 1.0F).score(1.0F, 1) : 1.0F; - EXPECT_FLOAT_EQ(score, expected_score) << pattern << " doc=" << doc; - } - } } } } diff --git a/be/test/storage/index/inverted/query/engine/regexp_query_test.cpp b/be/test/storage/index/inverted/query/engine/regexp_query_test.cpp index 86907a49fe19df..68fba5ef614db4 100644 --- a/be/test/storage/index/inverted/query/engine/regexp_query_test.cpp +++ b/be/test/storage/index/inverted/query/engine/regexp_query_test.cpp @@ -50,15 +50,13 @@ class RegexpQueryV2Test : public testing::Test { st = io::global_local_filesystem()->create_directory(kTestDir); ASSERT_TRUE(st.ok()) << st; std::string field_name = "content"; - create_test_index(field_name, kTestDir, - {"apple123", "apple456", "banana789", - "test123abc", "pattern456", "regex999", - "match123", "search456", "prefix123", - "suffix789", "apple_test", "banana_data", - "test_pattern", "data_regex", "apple_banana", - "test_match", "pattern_test", "prefix_suffix", - "abc123xyz", "def456ghi", "apple789 apple789 apple456"}, - "standard"); + create_test_index( + field_name, kTestDir, + {"apple123", "apple456", "banana789", "test123abc", "pattern456", + "regex999", "match123", "search456", "prefix123", "suffix789", + "apple_test", "banana_data", "test_pattern", "data_regex", "apple_banana", + "test_match", "pattern_test", "prefix_suffix", "abc123xyz", "def456ghi"}, + "standard"); } void TearDown() override { @@ -182,36 +180,6 @@ TEST_F(RegexpQueryV2Test, test_regexp_query_with_scoring) { ASSERT_NE(regexp_weight, nullptr); } -TEST_F(RegexpQueryV2Test, test_regexp_query_scores_matching_terms) { - auto context = std::make_shared(); - context->collection_statistics = std::make_shared(); - context->collection_similarity = std::make_shared(); - - auto* dir = FSDirectory::getDirectory(kTestDir.c_str()); - auto reader_holder = make_shared_reader(lucene::index::IndexReader::open(dir, true)); - ASSERT_NE(reader_holder, nullptr); - - std::wstring field = StringHelper::to_wstring("content"); - auto query = std::make_shared( - context, field, index_query::TermPatternKind::kRegexp, anchored("apple.*")); - auto weight = query->weight(true); - - query_v2::QueryExecutionContext exec_ctx; - exec_ctx.segment_num_rows = reader_holder->maxDoc(); - exec_ctx.sources = {clucene_index_source(reader_holder, field, nullptr)}; - exec_ctx.field_sources.emplace(field, exec_ctx.sources.front()); - - auto scorer = weight->scorer(exec_ctx); - ASSERT_EQ(scorer->seek(0), 0); - float single_term_score = scorer->score(); - EXPECT_NE(single_term_score, 1.0F); - uint32_t last_doc = reader_holder->maxDoc() - 1; - ASSERT_EQ(scorer->seek(last_doc), last_doc); - EXPECT_GT(scorer->score(), single_term_score); - - _CLDECDELETE(dir); -} - // Test regexp query execution TEST_F(RegexpQueryV2Test, test_regexp_query_execution) { auto context = std::make_shared(); @@ -660,7 +628,7 @@ TEST_F(RegexpQueryV2Test, test_whole_term_match_any_term) { doc = scorer->advance(); } - EXPECT_EQ(result.cardinality(), 21); + EXPECT_EQ(result.cardinality(), 20); _CLDECDELETE(dir); } diff --git a/be/test/storage/index/inverted/query/engine/wildcard_query_test.cpp b/be/test/storage/index/inverted/query/engine/wildcard_query_test.cpp index 847cbbe8b2c003..0262023f75f32f 100644 --- a/be/test/storage/index/inverted/query/engine/wildcard_query_test.cpp +++ b/be/test/storage/index/inverted/query/engine/wildcard_query_test.cpp @@ -59,20 +59,14 @@ class WildcardQueryV2Test : public testing::Test { private: void create_test_index(const std::string& field_name, const std::string& dir) { - std::vector test_data = {"apple", "application", "apply", - "apricot", "banana", "band", - "bandana", "can", "candy", - "cat", "catalog", "dog", - "document", "door", "hello", - "help", "helpful", "test", - "testing", "testcase", "wildcard", - "wild", "wilderness", "card", - "cardboard", "abc", "abcd", - "abcde", "prefix123", "123suffix", - "pre123", "123suf", "both123both", - "star", "start", "started", - "starter", "question", "quest", - "query", "queries", "test test testing"}; + std::vector test_data = { + "apple", "application", "apply", "apricot", "banana", "band", + "bandana", "can", "candy", "cat", "catalog", "dog", + "document", "door", "hello", "help", "helpful", "test", + "testing", "testcase", "wildcard", "wild", "wilderness", "card", + "cardboard", "abc", "abcd", "abcde", "prefix123", "123suffix", + "pre123", "123suf", "both123both", "star", "start", "started", + "starter", "question", "quest", "query", "queries"}; CustomAnalyzerConfig::Builder builder; builder.with_tokenizer_config("standard", {}); @@ -165,36 +159,6 @@ TEST_F(WildcardQueryV2Test, test_wildcard_query_with_scoring) { ASSERT_NE(wildcard_weight, nullptr); } -TEST_F(WildcardQueryV2Test, test_wildcard_query_scores_matching_terms) { - auto context = std::make_shared(); - context->collection_statistics = std::make_shared(); - context->collection_similarity = std::make_shared(); - - auto* dir = FSDirectory::getDirectory(kTestDir.c_str()); - auto reader_holder = make_shared_reader(lucene::index::IndexReader::open(dir, true)); - ASSERT_NE(reader_holder, nullptr); - - std::wstring field = StringHelper::to_wstring("field"); - auto query = std::make_shared( - context, field, index_query::TermPatternKind::kWildcard, "test*"); - auto weight = query->weight(true); - - query_v2::QueryExecutionContext exec_ctx; - exec_ctx.segment_num_rows = reader_holder->maxDoc(); - exec_ctx.sources = {clucene_index_source(reader_holder, field, nullptr)}; - exec_ctx.field_sources.emplace(field, exec_ctx.sources.front()); - - auto scorer = weight->scorer(exec_ctx); - ASSERT_EQ(scorer->seek(17), 17); - float single_term_score = scorer->score(); - EXPECT_NE(single_term_score, 1.0F); - uint32_t last_doc = reader_holder->maxDoc() - 1; - ASSERT_EQ(scorer->seek(last_doc), last_doc); - EXPECT_GT(scorer->score(), single_term_score); - - _CLDECDELETE(dir); -} - // Test wildcard query with asterisk prefix pattern TEST_F(WildcardQueryV2Test, test_wildcard_query_prefix_pattern) { auto context = std::make_shared(); @@ -576,4 +540,4 @@ TEST_F(WildcardQueryV2Test, test_wildcard_query_move_semantics) { ASSERT_NE(weight2, nullptr); } -} // namespace doris::segment_v2 +} // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/test/storage/index/inverted/reader/collection_statistics_test.cpp b/be/test/storage/index/inverted/reader/collection_statistics_test.cpp index 6e00b652fd3e5b..917950e7b62e31 100644 --- a/be/test/storage/index/inverted/reader/collection_statistics_test.cpp +++ b/be/test/storage/index/inverted/reader/collection_statistics_test.cpp @@ -2559,6 +2559,52 @@ TEST_F(CollectionStatisticsTest, CollectSkipsIndexWithoutSimilarityScore) { EXPECT_TRUE(collect_infos.empty()); } +// A regexp scores a constant, so it collects no statistics even on an index whose MATCH_ANY does: +// analyzing its pattern 'ap.*' would register the made-up term "ap". +TEST_F(CollectionStatisticsTest, RegexpCollectsNoStatistics) { + auto tablet_schema = std::make_shared(); + + constexpr int32_t kColUid = 1360; + TabletColumn col; + col.set_unique_id(kColUid); + col.set_name("body"); + col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(col); + + TabletIndexPB index_pb; + index_pb.set_index_id(2360); + index_pb.set_index_name("body_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kColUid); + (*index_pb.mutable_properties())["parser"] = "english"; + (*index_pb.mutable_properties())["support_phrase"] = "true"; + TabletIndex index; + index.init_from_pb(index_pb); + tablet_schema->append_index(std::move(index)); + + constexpr int kSlotId = 86; + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kColUid, "body", {}); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->_children.push_back( + std::make_shared("body", SlotId(kSlotId))); + match_expr->_children.push_back(std::make_shared("ap.*")); + + MatchPredicateCollector collector; + std::unordered_map collect_infos; + match_expr->set_opcode(TExprOpcode::MATCH_ANY); + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + ASSERT_TRUE(status.ok()) << status.msg(); + EXPECT_EQ(collect_infos.size(), 1U); + + collect_infos.clear(); + match_expr->set_opcode(TExprOpcode::MATCH_REGEXP); + status = collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + ASSERT_TRUE(status.ok()) << status.msg(); + EXPECT_TRUE(collect_infos.empty()); +} + TEST_F(CollectionStatisticsTest, CollectPreservesLogicalClauseShapesForSameFieldName) { auto tablet_schema = std::make_shared(); diff --git a/be/test/storage/segment/index_reader_helper_test.cpp b/be/test/storage/segment/index_reader_helper_test.cpp index 64d0c173858c2f..4e57a23f174974 100644 --- a/be/test/storage/segment/index_reader_helper_test.cpp +++ b/be/test/storage/segment/index_reader_helper_test.cpp @@ -464,6 +464,32 @@ TEST_F(IndexReaderHelperTest, IsNeedSimilarityScoreWithTExprOpcodeTest) { EXPECT_FALSE(IndexReaderHelper::is_need_similarity_score(TExprOpcode::LT, &index_meta)); } +TEST_F(IndexReaderHelperTest, RegexpScoresConstantOnEveryIndex) { + // A regexp scores a constant 1 and reads no statistics, so neither the analyzer nor phrase + // support decides whether it scores. + for (const auto& properties : std::vector> { + {}, + {{"parser", "none"}}, + {{"parser", "english"}}, + {{"parser", "english"}, {"support_phrase", "true"}}, + {{"normalizer", "split_norm"}}}) { + SCOPED_TRACE(::testing::PrintToString(properties)); + TabletIndexPB pb; + pb.set_index_id(1); + pb.set_index_name("regexp_index"); + pb.set_index_type(IndexType::INVERTED); + pb.add_col_unique_id(1); + for (const auto& [key, value] : properties) { + (*pb.mutable_properties())[key] = value; + } + TabletIndex index; + index.init_from_pb(pb); + EXPECT_TRUE(IndexReaderHelper::is_need_similarity_score(TExprOpcode::MATCH_REGEXP, &index)); + EXPECT_TRUE(IndexReaderHelper::is_need_similarity_score( + InvertedIndexQueryType::MATCH_REGEXP_QUERY, &index)); + } +} + TEST_F(IndexReaderHelperTest, MatchScoringRequiresAnalyzedPhraseIndex) { struct Case { std::map properties; diff --git a/regression-test/data/search/test_search_score_pattern_constant.out b/regression-test/data/search/test_search_score_pattern_constant.out new file mode 100644 index 00000000000000..e8603f53e4f5e7 --- /dev/null +++ b/regression-test/data/search/test_search_score_pattern_constant.out @@ -0,0 +1,124 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !V2_search_regexp -- +2 1 +3 1 +6 1 + +-- !V2_search_wildcard -- +2 1 +3 1 +6 1 + +-- !V2_search_prefix -- +2 1 +3 1 +6 1 + +-- !V2_search_keyword_regexp -- +6 1 +7 1 + +-- !V2_match_regexp -- +2 1 +3 1 +6 1 + +-- !V2_match_regexp_keyword -- +6 1 +7 1 + +-- !V2_match_and_match_regexp -- +2 2.023573 + +-- !V2_search_and_regexp -- +2 2.023573 + +-- !V2_match_only -- +2 1.023573 +5 1.163151 + +-- !V2_match_and_equality -- +2 1.023573 + +-- !V3_search_regexp -- +2 1 +3 1 +6 1 + +-- !V3_search_wildcard -- +2 1 +3 1 +6 1 + +-- !V3_search_prefix -- +2 1 +3 1 +6 1 + +-- !V3_search_keyword_regexp -- +6 1 +7 1 + +-- !V3_match_regexp -- +2 1 +3 1 +6 1 + +-- !V3_match_regexp_keyword -- +6 1 +7 1 + +-- !V3_match_and_match_regexp -- +2 2.023573 + +-- !V3_search_and_regexp -- +2 2.023573 + +-- !V3_match_only -- +2 1.023573 +5 1.163151 + +-- !V3_match_and_equality -- +2 1.023573 + +-- !SNII_search_regexp -- +2 1 +3 1 +6 1 + +-- !SNII_search_wildcard -- +2 1 +3 1 +6 1 + +-- !SNII_search_prefix -- +2 1 +3 1 +6 1 + +-- !SNII_search_keyword_regexp -- +6 1 +7 1 + +-- !SNII_match_regexp -- +2 1 +3 1 +6 1 + +-- !SNII_match_regexp_keyword -- +6 1 +7 1 + +-- !SNII_match_and_match_regexp -- +2 2.023573 + +-- !SNII_search_and_regexp -- +2 2.023573 + +-- !SNII_match_only -- +2 1.023573 +5 1.163151 + +-- !SNII_match_and_equality -- +2 1.023573 + diff --git a/regression-test/suites/search/test_search_score_pattern_constant.groovy b/regression-test/suites/search/test_search_score_pattern_constant.groovy new file mode 100644 index 00000000000000..f3cf96158509c7 --- /dev/null +++ b/regression-test/suites/search/test_search_score_pattern_constant.groovy @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// score() of pattern queries follows Elasticsearch's default rewrite: a regexp, wildcard or +// prefix in SEARCH and a MATCH_REGEXP score a constant 1 per matching row, whatever segment the +// row is in, and add exactly 1 next to a scored clause. An equality predicate adds nothing. +suite("test_search_score_pattern_constant", "p0") { + // SEARCH and MATCH need index execution even when CI fuzzes these session defaults. + sql "set enable_common_expr_pushdown = true" + sql "set enable_inverted_index_query = true" + + for (def storageFormat : ["V2", "V3", "SNII"]) { + sql "DROP TABLE IF EXISTS test_search_score_pattern_constant" + sql """ + CREATE TABLE test_search_score_pattern_constant ( + id BIGINT NOT NULL, + content STRING NULL, + tag STRING NULL, + INDEX idx_content(content) USING INVERTED + PROPERTIES("parser" = "english", "support_phrase" = "true"), + INDEX idx_tag(tag) USING INVERTED + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default:1", + "inverted_index_storage_format" = "${storageFormat}", + "disable_auto_compaction" = "true" + ) + """ + // Two inserts make two segments with different term statistics; a constant score must not + // depend on them. + sql """INSERT INTO test_search_score_pattern_constant VALUES + (1, 'Apache Doris is a high performance analytical database', 'doris'), + (2, 'Inverted indexes are suitable for full text search', 'index'), + (3, 'The parser splits text into multiple terms', 'parser'), + (4, 'Doris supports fast real time data analysis', 'doris'), + (5, 'Users can quickly search product names', 'search') + """ + sql """INSERT INTO test_search_score_pattern_constant VALUES + (6, 'text text text', 'text'), (7, 'plain words only', 'textual') + """ + sql "sync" + + for (def query : [ + ["search_regexp", "SEARCH('content:/.*ext/')"], + ["search_wildcard", "SEARCH('content:*ext')"], + ["search_prefix", "SEARCH('content:te*')"], + ["search_keyword_regexp", "SEARCH('tag:/text.*/')"], + ["match_regexp", "content MATCH_REGEXP 'ext'"], + ["match_regexp_keyword", "tag MATCH_REGEXP 'text'"], + // A pattern next to a scored clause adds exactly 1, as an Elasticsearch bool must + // with a regexp clause does. + ["match_and_match_regexp", "content MATCH 'search' AND content MATCH_REGEXP 'ext'"], + ["search_and_regexp", "SEARCH('content:search AND content:/.*ext/')"], + ["match_only", "content MATCH 'search'"], + ["match_and_equality", "content MATCH 'search' AND tag = 'index'"]]) { + quickTest("${storageFormat}_${query[0]}", """ + SELECT id, ROUND(s, 6) FROM ( + SELECT id, score() AS s FROM test_search_score_pattern_constant + WHERE ${query[1]} ORDER BY s DESC LIMIT 10 + ) hits ORDER BY id + """, true) + } + } +}