Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,18 @@
#include <gen_cpp/PaloBrokerService_types.h>

#include <algorithm>
#include <cmath>
#include <string>
#include <vector>

#include "storage/index/inverted/query_v2/bit_set_query/bit_set_scorer.h"
#include "storage/index/inverted/query_v2/const_score_query/const_score_scorer.h"
#include "storage/index/inverted/query_v2/nullable_scorer.h"
#include "storage/index/inverted/query_v2/score_combiner.h"
#include "storage/index/inverted/query_v2/segment_postings.h"
#include "storage/index/inverted/query_v2/term_query/term_scorer.h"
#include "storage/index/inverted/query_v2/union/buffered_union.h"
#include "storage/index/inverted/similarity/bm25_similarity.h"
#include "storage/index/inverted/util/string_helper.h"
#include "util/hyperscan_util.h"

Expand Down Expand Up @@ -110,6 +116,10 @@ ScorerPtr RegexpWeight::regexp_scorer(const QueryExecutionContext& context,
}

auto reader = lookup_reader(_field, context, binding_key);
if (_enable_scoring) {
return scored_terms_scorer(reader.get(), context, binding_key, matching_terms);
}

auto doc_bitset = std::make_shared<roaring::Roaring>();
for (const auto& term : matching_terms) {
auto segment_postings =
Expand All @@ -126,6 +136,46 @@ ScorerPtr RegexpWeight::regexp_scorer(const QueryExecutionContext& context,
return const_score;
}

ScorerPtr RegexpWeight::scored_terms_scorer(lucene::index::IndexReader* reader,
const QueryExecutionContext& context,
const std::string& binding_key,
const std::vector<std::wstring>& matching_terms) {
// Expanded terms are not collected in collection statistics, so use reader statistics.
const auto num_docs = reader->maxDoc();
DORIS_CHECK_GT(num_docs, 0);
const auto total_term_freq = reader->sumTotalTermFreq(_field.c_str());
// Without norms, postings use a document length of one.
const float avgdl = static_cast<float>(total_term_freq.value_or(num_docs)) / num_docs;
DORIS_CHECK_GT(avgdl, 0.0F);

std::vector<ScorerPtr> term_scorers;
term_scorers.reserve(matching_terms.size());
auto logical_field = logical_field_or_fallback(context, binding_key, _field);
for (const auto& term : matching_terms) {
Term lucene_term(_field.c_str(), term.c_str());
const int32_t doc_freq = reader->docFreq(&lucene_term);
if (doc_freq == 0) {
continue;
}
const auto idf = static_cast<float>(
std::log1p((static_cast<double>(num_docs) - doc_freq + 0.5) / (doc_freq + 0.5)));
auto similarity = std::make_shared<BM25Similarity>(idf, avgdl);
auto posting =
create_term_posting(reader, _field, term, true, similarity, _context->io_ctx);
if (posting) {
term_scorers.emplace_back(std::make_shared<TermScorer>(
std::move(posting), std::move(similarity), logical_field));
}
}
if (term_scorers.empty()) {
return std::make_shared<EmptyScorer>();
}
if (term_scorers.size() == 1) {
return std::move(term_scorers.front());
}
return make_buffered_union(term_scorers, std::make_shared<SumCombiner>());
}

std::optional<std::string> RegexpWeight::get_regex_prefix(const std::string& pattern) {
DBUG_EXECUTE_IF("RegexpQuery.get_regex_prefix", { return std::nullopt; });

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ class RegexpWeight : public Weight {

private:
ScorerPtr regexp_scorer(const QueryExecutionContext& context, const std::string& binding_key);
ScorerPtr scored_terms_scorer(lucene::index::IndexReader* reader,
const QueryExecutionContext& context,
const std::string& binding_key,
const std::vector<std::wstring>& matching_terms);

std::optional<std::string> get_regex_prefix(const std::string& pattern);
void collect_matching_terms(const QueryExecutionContext& context,
Expand All @@ -46,7 +50,7 @@ class RegexpWeight : public Weight {

std::wstring _field;
std::string _pattern;
[[maybe_unused]] bool _enable_scoring = false;
bool _enable_scoring = false;
bool _nullable = true;
// Set to 0 to disable limit (ES has no default limit for prefix queries)
// The limit prevents collecting too many terms, but can cause incorrect results
Expand Down
61 changes: 56 additions & 5 deletions be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include "storage/index/index_query_context.h"
#include "storage/index/inverted/analyzer/custom_analyzer.h"
#include "storage/index/inverted/query_v2/regexp_query/regexp_weight.h"
#include "storage/index/inverted/query_v2/term_query/term_query.h"
#include "storage/index/inverted/util/string_helper.h"

CL_NS_USE(search)
Expand Down Expand Up @@ -58,10 +59,13 @@ class RegexpQueryV2Test : public testing::Test {
private:
void create_test_index(const std::string& field_name, const std::string& dir) {
std::vector<std::string> test_data = {
"apple123", "apple456", "banana789", "test123abc", "pattern456",
"regex999", "match123", "search456", "prefix123", "suffix789",
"apple_test", "banana_data", "test_pattern", "data_regex", "apple_banana",
"test_match", "pattern_test", "prefix_suffix", "abc123xyz", "def456ghi"};
"apple123", "apple456", "banana789",
"test123abc", "pattern456", "regex999",
"match123", "search456", "prefix123",
"suffix789", "apple_test", "banana_data",
"test_pattern", "data_regex", "apple_banana",
"test_match", "pattern_test", "prefix_suffix",
"abc123xyz", "def456ghi", "apple789 apple789 apple456"};

CustomAnalyzerConfig::Builder builder;
builder.with_tokenizer_config("standard", {});
Expand Down Expand Up @@ -167,6 +171,53 @@ TEST_F(RegexpQueryV2Test, test_regexp_query_with_scoring) {
ASSERT_NE(regexp_weight, nullptr);
}

TEST_F(RegexpQueryV2Test, test_regexp_query_scores_matching_terms) {
auto context = std::make_shared<IndexQueryContext>();
context->collection_statistics = std::make_shared<CollectionStatistics>();
context->collection_similarity = std::make_shared<CollectionSimilarity>();

auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
auto reader_holder = make_shared_reader(lucene::index::IndexReader::open(dir, true));
ASSERT_NE(reader_holder, nullptr);

std::wstring field = StringHelper::to_wstring("content");
auto query = std::make_shared<query_v2::RegexpQuery>(context, field, "apple.*");
auto weight = query->weight(true);

query_v2::QueryExecutionContext exec_ctx;
exec_ctx.segment_num_rows = reader_holder->maxDoc();
exec_ctx.readers = {reader_holder};
exec_ctx.field_reader_bindings.emplace(field, reader_holder);

auto statistics = context->collection_statistics;
statistics->_total_num_docs = reader_holder->maxDoc();
statistics->_total_num_tokens[field] =
reader_holder->sumTotalTermFreq(field.c_str()).value_or(reader_holder->maxDoc());
for (const auto* term : {L"apple123", L"apple789", L"apple456"}) {
Term lucene_term(field.c_str(), term);
statistics->_term_doc_freqs[field][term] = reader_holder->docFreq(&lucene_term);
}
auto term_score = [&](const wchar_t* term, uint32_t doc_id) {
auto term_query = std::make_shared<query_v2::TermQuery>(context, field, term);
auto term_scorer = term_query->weight(true)->scorer(exec_ctx);
EXPECT_EQ(term_scorer->seek(doc_id), doc_id);
return term_scorer->score();
};

auto scorer = weight->scorer(exec_ctx);
ASSERT_EQ(scorer->seek(0), 0);
float single_term_score = scorer->score();
EXPECT_NE(single_term_score, 1.0F);
EXPECT_FLOAT_EQ(single_term_score, term_score(L"apple123", 0));
uint32_t last_doc = reader_holder->maxDoc() - 1;
ASSERT_EQ(scorer->seek(last_doc), last_doc);
EXPECT_GT(scorer->score(), single_term_score);
EXPECT_NEAR(scorer->score(),
term_score(L"apple789", last_doc) + term_score(L"apple456", last_doc), 1e-5F);

_CLDECDELETE(dir);
}

// Test regexp query execution
TEST_F(RegexpQueryV2Test, test_regexp_query_execution) {
auto context = std::make_shared<IndexQueryContext>();
Expand Down Expand Up @@ -574,7 +625,7 @@ TEST_F(RegexpQueryV2Test, test_make_exact_match_wildcard_pattern) {
doc = scorer->advance();
}

EXPECT_EQ(result.cardinality(), 20);
EXPECT_EQ(result.cardinality(), 21);

_CLDECDELETE(dir);
}
Expand Down
72 changes: 63 additions & 9 deletions be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include "common/status.h"
#include "storage/index/index_query_context.h"
#include "storage/index/inverted/analyzer/custom_analyzer.h"
#include "storage/index/inverted/query_v2/term_query/term_query.h"
#include "storage/index/inverted/query_v2/wildcard_query/wildcard_weight.h"
#include "storage/index/inverted/util/string_helper.h"

Expand Down Expand Up @@ -57,14 +58,20 @@ class WildcardQueryV2Test : public testing::Test {

private:
void create_test_index(const std::string& field_name, const std::string& dir) {
std::vector<std::string> test_data = {
"apple", "application", "apply", "apricot", "banana", "band",
"bandana", "can", "candy", "cat", "catalog", "dog",
"document", "door", "hello", "help", "helpful", "test",
"testing", "testcase", "wildcard", "wild", "wilderness", "card",
"cardboard", "abc", "abcd", "abcde", "prefix123", "123suffix",
"pre123", "123suf", "both123both", "star", "start", "started",
"starter", "question", "quest", "query", "queries"};
std::vector<std::string> test_data = {"apple", "application", "apply",
"apricot", "banana", "band",
"bandana", "can", "candy",
"cat", "catalog", "dog",
"document", "door", "hello",
"help", "helpful", "test",
"testing", "testcase", "wildcard",
"wild", "wilderness", "card",
"cardboard", "abc", "abcd",
"abcde", "prefix123", "123suffix",
"pre123", "123suf", "both123both",
"star", "start", "started",
"starter", "question", "quest",
"query", "queries", "test test testing"};

CustomAnalyzerConfig::Builder builder;
builder.with_tokenizer_config("standard", {});
Expand Down Expand Up @@ -155,6 +162,53 @@ TEST_F(WildcardQueryV2Test, test_wildcard_query_with_scoring) {
ASSERT_NE(wildcard_weight, nullptr);
}

TEST_F(WildcardQueryV2Test, test_wildcard_query_scores_matching_terms) {
auto context = std::make_shared<IndexQueryContext>();
context->collection_statistics = std::make_shared<CollectionStatistics>();
context->collection_similarity = std::make_shared<CollectionSimilarity>();

auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
auto reader_holder = make_shared_reader(lucene::index::IndexReader::open(dir, true));
ASSERT_NE(reader_holder, nullptr);

std::wstring field = StringHelper::to_wstring("field");
auto query = std::make_shared<query_v2::WildcardQuery>(context, field, "test*");
auto weight = query->weight(true);

query_v2::QueryExecutionContext exec_ctx;
exec_ctx.segment_num_rows = reader_holder->maxDoc();
exec_ctx.readers = {reader_holder};
exec_ctx.field_reader_bindings.emplace(field, reader_holder);

auto statistics = context->collection_statistics;
statistics->_total_num_docs = reader_holder->maxDoc();
statistics->_total_num_tokens[field] =
reader_holder->sumTotalTermFreq(field.c_str()).value_or(reader_holder->maxDoc());
for (const auto* term : {L"test", L"testing"}) {
Term lucene_term(field.c_str(), term);
statistics->_term_doc_freqs[field][term] = reader_holder->docFreq(&lucene_term);
}
auto term_score = [&](const wchar_t* term, uint32_t doc_id) {
auto term_query = std::make_shared<query_v2::TermQuery>(context, field, term);
auto term_scorer = term_query->weight(true)->scorer(exec_ctx);
EXPECT_EQ(term_scorer->seek(doc_id), doc_id);
return term_scorer->score();
};

auto scorer = weight->scorer(exec_ctx);
ASSERT_EQ(scorer->seek(17), 17);
float single_term_score = scorer->score();
EXPECT_NE(single_term_score, 1.0F);
EXPECT_FLOAT_EQ(single_term_score, term_score(L"test", 17));
uint32_t last_doc = reader_holder->maxDoc() - 1;
ASSERT_EQ(scorer->seek(last_doc), last_doc);
EXPECT_GT(scorer->score(), single_term_score);
EXPECT_NEAR(scorer->score(), term_score(L"test", last_doc) + term_score(L"testing", last_doc),
1e-5F);

_CLDECDELETE(dir);
}

// Test wildcard query with asterisk prefix pattern
TEST_F(WildcardQueryV2Test, test_wildcard_query_prefix_pattern) {
auto context = std::make_shared<IndexQueryContext>();
Expand Down Expand Up @@ -524,4 +578,4 @@ TEST_F(WildcardQueryV2Test, test_wildcard_query_move_semantics) {
ASSERT_NE(weight2, nullptr);
}

} // namespace doris::segment_v2
} // namespace doris::segment_v2
Loading
Loading