Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions be/src/storage/index/index_reader_helper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ bool can_score_match(const TabletIndex* index_meta) {

bool IndexReaderHelper::is_need_similarity_score(InvertedIndexQueryType query_type,
const TabletIndex* index_meta) {
// A regexp scores a constant 1 per matching row, as an Elasticsearch regexp query does in
// query context. The constant reads no statistics, norms or positions, so every index can
// score it: body MATCH 'apple' AND body MATCH_REGEXP 'ap.*' adds 1 to the apple score.
if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) {
return true;
}
return (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY ||
Expand All @@ -45,6 +51,10 @@ bool IndexReaderHelper::is_need_similarity_score(InvertedIndexQueryType query_ty

bool IndexReaderHelper::is_need_similarity_score(TExprOpcode::type query_type,
const TabletIndex* index_meta) {
// See the InvertedIndexQueryType overload: a regexp scores a constant on every index.
if (query_type == TExprOpcode::MATCH_REGEXP) {
return true;
}
return (query_type == TExprOpcode::MATCH_ANY || query_type == TExprOpcode::MATCH_ALL ||
query_type == TExprOpcode::MATCH_PHRASE ||
query_type == TExprOpcode::MATCH_PHRASE_PREFIX) &&
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@ class ExpandQuery : public Query {
_pattern(std::move(pattern)) {}
~ExpandQuery() override = default;

WeightPtr weight(bool enable_scoring) override {
return std::make_shared<ExpandWeight>(_context, _field, _kind, _pattern, enable_scoring);
// Scores are constant, so scoring changes nothing.
WeightPtr weight(bool /*enable_scoring*/) override {
return std::make_shared<ExpandWeight>(_context, _field, _kind, _pattern);
}

private:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

#include "storage/index/inverted/query/engine/expand_query/expand_weight.h"

#include <cmath>
#include <memory>
#include <roaring/roaring.hh>
#include <vector>
Expand All @@ -26,14 +25,10 @@
#include "storage/index/inverted/query/engine/bit_set_query/bit_set_scorer.h"
#include "storage/index/inverted/query/engine/const_score_query/const_score_scorer.h"
#include "storage/index/inverted/query/engine/nullable_scorer.h"
#include "storage/index/inverted/query/engine/score_combiner.h"
#include "storage/index/inverted/query/engine/term_query/term_scorer.h"
#include "storage/index/inverted/query/engine/union/buffered_union.h"
#include "storage/index/inverted/query/exec/block_doc_set.h"
#include "storage/index/inverted/query/exec/collect_postings.h"
#include "storage/index/inverted/query/exec/roaring_docid_sink.h"
#include "storage/index/inverted/query/exec/term_waves.h"
#include "storage/index/inverted/query/scoring/bm25_similarity.h"
#include "storage/index/inverted/query/spi/postings_cursor.h"

namespace doris::segment_v2::inverted_index::query_v2 {
Expand Down Expand Up @@ -63,15 +58,11 @@ Status collect_expanded_rows(index_query::IndexSource& source, index_query::Term
}

ExpandWeight::ExpandWeight(IndexQueryContextPtr context, std::wstring field,
index_query::TermPatternKind kind, std::string pattern,
bool enable_scoring)
index_query::TermPatternKind kind, std::string pattern)
: _context(std::move(context)),
_field(std::move(field)),
_kind(kind),
_pattern(std::move(pattern)),
// Prefix and phrase-edge expansions retain their constant-score behavior.
_enable_scoring(enable_scoring && (kind == index_query::TermPatternKind::kRegexp ||
kind == index_query::TermPatternKind::kWildcard)) {}
_pattern(std::move(pattern)) {}

std::shared_ptr<roaring::Roaring> ExpandWeight::_rows(const QueryExecutionContext& context,
const std::string& binding_key,
Expand All @@ -94,68 +85,12 @@ std::shared_ptr<roaring::Roaring> ExpandWeight::_rows(const QueryExecutionContex

ScorerPtr ExpandWeight::scorer(const QueryExecutionContext& context,
const std::string& binding_key) {
// The listing fast path discards term frequencies, so scored patterns need term scorers.
if (_enable_scoring) {
return make_nullable_scorer(_scored_terms(context, binding_key),
logical_field_or_fallback(context, binding_key, _field),
context.null_resolver);
}
auto scorer = std::make_shared<ConstScoreScorer<BitSetScorerPtr>>(
std::make_shared<BitSetScorer>(_rows(context, binding_key, nullptr)));
return make_nullable_scorer(scorer, logical_field_or_fallback(context, binding_key, _field),
context.null_resolver);
}

ScorerPtr ExpandWeight::_scored_terms(const QueryExecutionContext& context,
const std::string& binding_key) {
index_query::TermPattern pattern;
THROW_IF_ERROR(index_query::TermPattern::create(_kind, _pattern, &pattern));
auto source = lookup_source(_field, context, binding_key);
// An unbound field has no postings and contributes no matches.
if (source == nullptr) {
return std::make_shared<EmptyScorer>();
}
std::vector<std::string> terms;
THROW_IF_ERROR(source->expand_terms(
pattern, index_query::expansion_limit(_kind, index_query::max_expansions(*_context)),
&terms));
// A valid pattern may match no dictionary terms.
if (terms.empty()) {
return std::make_shared<EmptyScorer>();
}
// Expansion terms are absent from collection statistics; use the bound source's statistics.
index_query::IndexScoringStats stats;
THROW_IF_ERROR(source->scoring_stats(&stats));
DORIS_CHECK_GT(stats.doc_count, 0);
// Docs-only SNII still reports its physical token count. Only legacy sources without
// that metadata report zero, so use unit average length only for those sources.
const uint64_t token_count = stats.token_count != 0 ? stats.token_count : stats.doc_count;
const float avgdl = static_cast<float>(token_count) / stats.doc_count;
DORIS_CHECK_GT(avgdl, 0.0F);
std::vector<ScorerPtr> scorers;
scorers.reserve(terms.size());
auto logical_field = logical_field_or_fallback(context, binding_key, _field);
for (const auto& term : terms) {
uint64_t doc_freq = 0;
THROW_IF_ERROR(source->doc_freq(term, &doc_freq));
// The immutable source just enumerated this term and must have its postings.
DCHECK_GT(doc_freq, 0);
const auto idf = static_cast<float>(std::log1p(
(static_cast<double>(stats.doc_count) - doc_freq + 0.5) / (doc_freq + 0.5)));
auto similarity = std::make_shared<BM25Similarity>(idf, avgdl);
auto postings = open_postings(*source, term, /*positions=*/false, /*enable_scoring=*/true,
similarity);
DCHECK(postings != nullptr);
scorers.emplace_back(std::make_shared<TermScorer>(std::move(postings),
std::move(similarity), logical_field));
}
// One expanded term needs no union; multiple terms add their per-document BM25 scores.
if (scorers.size() == 1) {
return std::move(scorers.front());
}
return make_buffered_union(scorers, std::make_shared<SumCombiner>());
}

index_query::TruthSet ExpandWeight::listed_rows(const QueryExecutionContext& context,
const std::string& binding_key,
const roaring::Roaring* candidates) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,33 +29,30 @@ Status collect_expanded_rows(index_query::IndexSource& source, index_query::Term
int32_t max_expansions, const roaring::Roaring* candidates,
roaring::Roaring* rows);

// Every document that holds an expanded term; scored regexp and wildcard clauses sum BM25.
// Every document that holds a term the pattern expands to, with a constant score.
class ExpandWeight : public Weight {
public:
ExpandWeight(IndexQueryContextPtr context, std::wstring field,
index_query::TermPatternKind kind, std::string pattern,
bool enable_scoring = false);
index_query::TermPatternKind kind, std::string pattern);
~ExpandWeight() override = default;

ScorerPtr scorer(const QueryExecutionContext& context, const std::string& binding_key) override;

bool lists_rows(const QueryExecutionContext&, const std::string&) const override {
return !_enable_scoring;
return true;
}
index_query::TruthSet listed_rows(const QueryExecutionContext& context,
const std::string& binding_key,
const roaring::Roaring* candidates) override;

private:
ScorerPtr _scored_terms(const QueryExecutionContext& context, const std::string& binding_key);
std::shared_ptr<roaring::Roaring> _rows(const QueryExecutionContext& context,
const std::string& binding_key,
const roaring::Roaring* candidates);
IndexQueryContextPtr _context;
std::wstring _field;
index_query::TermPatternKind _kind;
std::string _pattern;
bool _enable_scoring;
};

} // namespace doris::segment_v2::inverted_index::query_v2
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,11 @@ Status MatchPredicateCollector::collect(RuntimeState* state, const TabletSchemaS
!IndexReaderHelper::is_need_similarity_score(expr->op(), index_meta)) {
return Status::OK();
}
// A regexp scores a constant and reads no statistics. Its value is a pattern, not text, so
// analyzing it would collect statistics for made-up terms: 'ap.*' would become "ap".
if (expr->op() == TExprOpcode::MATCH_REGEXP) {
return Status::OK();
}

DORIS_CHECK(analyzer_ctx->analyzer_provider != nullptr);
auto options = DataTypeSerDe::get_default_format_options();
Expand Down
23 changes: 9 additions & 14 deletions be/test/exprs/function/search/function_search_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
#include <gtest/gtest.h>

#include <chrono>
#include <cmath>
#include <initializer_list>
#include <map>
#include <memory>
Expand Down Expand Up @@ -1881,8 +1880,6 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryExecutesSelectedSniiWildcardReader)
std::make_shared<RecordingNativeInvertedIndexReader>(&decoy_meta, decoy_file_reader);
auto selected_reader = std::make_shared<RecordingNativeInvertedIndexReader>(
&selected_meta, selected_file_reader);
selected_reader->fake_source->set_doc_count(4);
selected_reader->fake_source->set_scoring_stats({.doc_count = 4, .token_count = 3});
selected_reader->fake_source->add("alpha", {0, 2});
selected_reader->fake_source->add("gamma", {1});
decoy_reader->fake_source->add("alpha", {1});
Expand Down Expand Up @@ -1931,8 +1928,7 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryExecutesSelectedSniiWildcardReader)
binding_key);
ASSERT_NE(nullptr, scorer);
EXPECT_EQ(0U, scorer->doc());
// The source has three tokens over four rows, even without stored norms.
EXPECT_NEAR(0.88F * std::log(2.0F), scorer->score(), 1e-6F);
EXPECT_FLOAT_EQ(1.0F, scorer->score());
expect_bitmap_eq(collect_docs(scorer), {0, 2});
// The pattern was lowercased by the selected analyzer and enumerated on the selected
// index only; a leading wildcard has no prefix to enumerate from.
Expand Down Expand Up @@ -2526,8 +2522,7 @@ static void search_snii_pattern(const std::map<std::string, std::string>& proper
ASSERT_TRUE(status.ok()) << status.to_string();
EXPECT_EQ(0, reader->query_calls) << value;
// The scorer enumerates the stem on the source; the reader itself is never asked.
// This helper checks normalization and routing, without requesting scoring statistics.
auto weight = query->weight(false);
auto weight = query->weight(true);
ASSERT_NE(nullptr, weight);
auto scorer = weight->scorer(build_variant_search_query_execution_context(4, resolver, nullptr),
binding_key);
Expand Down Expand Up @@ -2760,12 +2755,13 @@ TEST_F(FunctionSearchTest, TestSniiScoredLeafRanksByTheEngineBm25) {
}
}

// Scored wildcard leaves read frequencies and statistics through the unified source,
// even when the old reader query API publishes no per-document scores.
TEST_F(FunctionSearchTest, TestSniiNativeWildcardUsesSourceScoringStatistics) {
// A leaf for which the reader publishes no per-document score must keep the constant that the
// CLucene path also gives its own constant-score leaves, so that routing scored clauses through a
// new scorer does not quietly re-rank the unscored ones. The fake reader is what withholds the
// scores here, so this pins the "nothing published -> BitSetQuery -> 1.0" behaviour; it does not
// pin which query types the production is_need_similarity_score gate rejects.
TEST_F(FunctionSearchTest, TestSniiNativeLeafWithoutPublishedScoresKeepsConstantScore) {
SniiScoringFixture fixture(43, 4);
fixture.reader->fake_source->set_doc_count(4);
fixture.reader->fake_source->set_scoring_stats({.doc_count = 4, .token_count = 2});
fixture.reader->fake_source->add("alpha", {0, 2});

inverted_index::query_v2::QueryPtr query;
Expand All @@ -2782,8 +2778,7 @@ TEST_F(FunctionSearchTest, TestSniiNativeWildcardUsesSourceScoringStatistics) {
auto scorer = weight->scorer(exec_ctx, binding_key);
ASSERT_NE(nullptr, scorer);
EXPECT_EQ(0U, scorer->doc());
// avgdl = 2/4 and implicit length = 1: BM25 is idf * 2.2 / 3.1.
EXPECT_NEAR((2.2F / 3.1F) * std::log(2.0F), scorer->score(), 1e-6F);
EXPECT_FLOAT_EQ(1.0F, scorer->score());
expect_bitmap_eq(collect_docs(scorer), {0, 2});
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
#include <CLucene.h>
#include <gtest/gtest.h>

#include <cmath>
#include <memory>
#include <roaring/roaring.hh>
#include <string>
Expand All @@ -33,8 +32,6 @@
#include "storage/index/inverted/inverted_index_reader.h"
#include "storage/index/inverted/query/engine/expand_query/expand_query.h"
#include "storage/index/inverted/query/logical/node.h"
#include "storage/index/inverted/query/scoring/bm25_similarity.h"
#include "storage/index/inverted/query/scoring/collection_similarity.h"
#include "storage/index/inverted/query/spi/fake_index_source.h"
#include "storage/olap_common.h"

Expand Down Expand Up @@ -495,8 +492,6 @@ TEST(ExpansionLeafTest, KeepsPatternLimitsAndLeavesRootCandidatesToTheCaller) {
auto source = std::make_shared<index_query::testing::FakeIndexSource>();
source->batches = batches;
source->set_doc_count(10);
// Scored regexp and wildcard leaves need statistics from their bound source.
source->set_scoring_stats({.doc_count = 10, .token_count = 10});
source->add("apple", {0, 4});
source->add("application", {1});
source->add("apply", {2, 4});
Expand All @@ -514,30 +509,12 @@ TEST(ExpansionLeafTest, KeepsPatternLimitsAndLeavesRootCandidatesToTheCaller) {
expected.add(2);
}
for (const bool scoring : {false, true}) {
context->collection_similarity =
scoring ? std::make_shared<CollectionSimilarity>() : nullptr;
auto result = std::make_shared<roaring::Roaring>();
result->add(9);
const auto status = run_leaf(context, L"content", leaf, &candidates, scoring,
source, source->doc_count(), result);
ASSERT_TRUE(status.ok()) << status;
EXPECT_EQ(*result, expected) << pattern << " scoring=" << scoring;
if (scoring) {
const auto& scores = context->collection_similarity->_bm25_scores;
EXPECT_EQ(scores.size(), expected.cardinality() - 1);
const bool bm25 = kind == logical::ExpandKind::kWildcard ||
kind == logical::ExpandKind::kRegexp;
for (const auto& [doc, score] : scores) {
// Row 9 predates this leaf and must not acquire a score from it.
EXPECT_NE(doc, 9U);
EXPECT_TRUE(expected.contains(doc));
const float df = doc == 1 ? 1.0F : 2.0F;
const float idf = std::log1p((10.0F - df + 0.5F) / (df + 0.5F));
const float expected_score =
bm25 ? BM25Similarity(idf, 1.0F).score(1.0F, 1) : 1.0F;
EXPECT_FLOAT_EQ(score, expected_score) << pattern << " doc=" << doc;
}
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,13 @@ class RegexpQueryV2Test : public testing::Test {
st = io::global_local_filesystem()->create_directory(kTestDir);
ASSERT_TRUE(st.ok()) << st;
std::string field_name = "content";
create_test_index(field_name, kTestDir,
{"apple123", "apple456", "banana789",
"test123abc", "pattern456", "regex999",
"match123", "search456", "prefix123",
"suffix789", "apple_test", "banana_data",
"test_pattern", "data_regex", "apple_banana",
"test_match", "pattern_test", "prefix_suffix",
"abc123xyz", "def456ghi", "apple789 apple789 apple456"},
"standard");
create_test_index(
field_name, kTestDir,
{"apple123", "apple456", "banana789", "test123abc", "pattern456",
"regex999", "match123", "search456", "prefix123", "suffix789",
"apple_test", "banana_data", "test_pattern", "data_regex", "apple_banana",
"test_match", "pattern_test", "prefix_suffix", "abc123xyz", "def456ghi"},
"standard");
}

void TearDown() override {
Expand Down Expand Up @@ -182,36 +180,6 @@ TEST_F(RegexpQueryV2Test, test_regexp_query_with_scoring) {
ASSERT_NE(regexp_weight, nullptr);
}

TEST_F(RegexpQueryV2Test, test_regexp_query_scores_matching_terms) {
auto context = std::make_shared<IndexQueryContext>();
context->collection_statistics = std::make_shared<CollectionStatistics>();
context->collection_similarity = std::make_shared<CollectionSimilarity>();

auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
auto reader_holder = make_shared_reader(lucene::index::IndexReader::open(dir, true));
ASSERT_NE(reader_holder, nullptr);

std::wstring field = StringHelper::to_wstring("content");
auto query = std::make_shared<query_v2::ExpandQuery>(
context, field, index_query::TermPatternKind::kRegexp, anchored("apple.*"));
auto weight = query->weight(true);

query_v2::QueryExecutionContext exec_ctx;
exec_ctx.segment_num_rows = reader_holder->maxDoc();
exec_ctx.sources = {clucene_index_source(reader_holder, field, nullptr)};
exec_ctx.field_sources.emplace(field, exec_ctx.sources.front());

auto scorer = weight->scorer(exec_ctx);
ASSERT_EQ(scorer->seek(0), 0);
float single_term_score = scorer->score();
EXPECT_NE(single_term_score, 1.0F);
uint32_t last_doc = reader_holder->maxDoc() - 1;
ASSERT_EQ(scorer->seek(last_doc), last_doc);
EXPECT_GT(scorer->score(), single_term_score);

_CLDECDELETE(dir);
}

// Test regexp query execution
TEST_F(RegexpQueryV2Test, test_regexp_query_execution) {
auto context = std::make_shared<IndexQueryContext>();
Expand Down Expand Up @@ -660,7 +628,7 @@ TEST_F(RegexpQueryV2Test, test_whole_term_match_any_term) {
doc = scorer->advance();
}

EXPECT_EQ(result.cardinality(), 21);
EXPECT_EQ(result.cardinality(), 20);

_CLDECDELETE(dir);
}
Expand Down
Loading
Loading