Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions be/src/storage/compaction/collection_statistics.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,8 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, int3
// them, together with the field's token count, in the norms. A segment written without
// norms would feed a zero avgdl, or rank its rows as zero-length documents next to the
// segments that have norms, so refuse to score the collection. An index that is not
// analyzed never writes norms and is left as it is.
// analyzed never writes norms; use an average length of one when its legacy
// metadata does not include the total term frequency.
const auto token_count = index_reader->sumTotalTermFreq(ws_field_name.c_str());
if (!token_count.has_value() &&
segment_v2::inverted_index::InvertedIndexAnalyzer::should_analyzer(
Expand All @@ -220,7 +221,7 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, int3
"for a variant path, when inverted_index_skip_norms_for_variant is on",
seg_path, StringHelper::to_string(ws_field_name));
}
_total_num_tokens[ws_field_name] += token_count.value_or(0);
_total_num_tokens[ws_field_name] += token_count.value_or(index_reader->maxDoc());

for (const auto& term_info : collect_info.term_infos) {
auto iter = TermIterator::create(io_ctx, false, index_reader, ws_field_name,
Expand Down
31 changes: 21 additions & 10 deletions be/src/storage/index/inverted/query_v2/segment_postings.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,19 +49,23 @@ class SegmentPostings : public Postings {
public:
using IterVariant = std::variant<std::monostate, TermDocsPtr, TermPositionsPtr>;

explicit SegmentPostings(TermDocsPtr iter, bool enable_scoring, SimilarityPtr similarity)
explicit SegmentPostings(TermDocsPtr iter, bool enable_scoring, SimilarityPtr similarity,
bool has_norms = true)
: _iter(std::move(iter)),
_enable_scoring(enable_scoring),
_has_norms(has_norms),
_similarity(std::move(similarity)) {
if (auto* p = std::get_if<TermDocsPtr>(&_iter)) {
_raw_iter = p->get();
}
_init_doc();
}

explicit SegmentPostings(TermPositionsPtr iter, bool enable_scoring, SimilarityPtr similarity)
explicit SegmentPostings(TermPositionsPtr iter, bool enable_scoring, SimilarityPtr similarity,
bool has_norms = true)
: _iter(std::move(iter)),
_enable_scoring(enable_scoring),
_has_norms(has_norms),
_has_positions(true),
_similarity(std::move(similarity)) {
if (auto* p = std::get_if<TermPositionsPtr>(&_iter)) {
Expand Down Expand Up @@ -120,7 +124,7 @@ class SegmentPostings : public Postings {
}

uint32_t norm() const override {
if (!_enable_scoring || !_block.norm_many || _cursor == 0) {
if (!_enable_scoring || !_has_norms || !_block.norm_many || _cursor == 0) {
return 1;
}
return (*_block.norm_many)[_cursor - 1];
Expand Down Expand Up @@ -194,24 +198,28 @@ class SegmentPostings : public Postings {
return _block_max_score_cache;
}
int32_t max_block_freq = _raw_iter->getMaxBlockFreq();
int32_t max_block_norm = _raw_iter->getMaxBlockNorm();
int32_t max_block_norm = _has_norms ? _raw_iter->getMaxBlockNorm() : 1;
if (max_block_freq >= 0 && max_block_norm >= 0) {
_block_max_score_cache = _similarity->score(static_cast<float>(max_block_freq),
static_cast<int64_t>(max_block_norm));
return _block_max_score_cache;
}
return _similarity->max_score();
return max_score();
}

float max_score() const {
if (!_enable_scoring || !_similarity) {
return std::numeric_limits<float>::max();
}
// Missing norms have a fixed length of one, including for WAND bounds.
if (!_has_norms) {
return _similarity->score(std::numeric_limits<float>::max(), 1);
}
return _similarity->max_score();
}

int32_t max_block_freq() const { return _raw_iter->getMaxBlockFreq(); }
int32_t max_block_norm() const { return _raw_iter->getMaxBlockNorm(); }
int32_t max_block_norm() const { return _has_norms ? _raw_iter->getMaxBlockNorm() : 1; }

private:
bool _refill() {
Expand Down Expand Up @@ -241,6 +249,7 @@ class SegmentPostings : public Postings {
lucene::index::TermDocs* _raw_iter = nullptr;
uint32_t _doc = TERMINATED;
bool _enable_scoring = false;
bool _has_norms = true;
bool _has_positions = false;

DocRange _block;
Expand All @@ -253,13 +262,15 @@ class SegmentPostings : public Postings {
using SegmentPostingsPtr = std::shared_ptr<SegmentPostings>;

inline SegmentPostingsPtr make_segment_postings(TermDocsPtr iter, bool enable_scoring,
SimilarityPtr similarity) {
return std::make_shared<SegmentPostings>(std::move(iter), enable_scoring, similarity);
SimilarityPtr similarity, bool has_norms = true) {
return std::make_shared<SegmentPostings>(std::move(iter), enable_scoring, similarity,
has_norms);
}

inline SegmentPostingsPtr make_segment_postings(TermPositionsPtr iter, bool enable_scoring,
SimilarityPtr similarity) {
return std::make_shared<SegmentPostings>(std::move(iter), enable_scoring, similarity);
SimilarityPtr similarity, bool has_norms = true) {
return std::make_shared<SegmentPostings>(std::move(iter), enable_scoring, similarity,
has_norms);
}

} // namespace doris::segment_v2::inverted_index::query_v2
8 changes: 6 additions & 2 deletions be/src/storage/index/inverted/query_v2/weight.h
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,9 @@ class Weight {
const io::IOContext* io_ctx) const {
auto t = make_term_ptr(field.c_str(), term.c_str());
auto iter = make_term_doc_ptr(reader, t.get(), enable_scoring, io_ctx);
return iter ? make_segment_postings(std::move(iter), enable_scoring, similarity) : nullptr;
return iter ? make_segment_postings(std::move(iter), enable_scoring, similarity,
reader->hasNorms(field.c_str()))
: nullptr;
}

SegmentPostingsPtr create_position_posting(lucene::index::IndexReader* reader,
Expand All @@ -168,7 +170,9 @@ class Weight {
const io::IOContext* io_ctx) const {
auto t = make_term_ptr(field.c_str(), term.c_str());
auto iter = make_term_positions_ptr(reader, t.get(), enable_scoring, io_ctx);
return iter ? make_segment_postings(std::move(iter), enable_scoring, similarity) : nullptr;
return iter ? make_segment_postings(std::move(iter), enable_scoring, similarity,
reader->hasNorms(field.c_str()))
: nullptr;
}
};

Expand Down
10 changes: 2 additions & 8 deletions be/src/storage/index/inverted/similarity/bm25_similarity.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,8 @@
namespace doris::segment_v2 {
#include "common/compile_check_begin.h"

/**
* BM25 similarity scoring implementation for inverted index queries.
*
* Note: Currently, BM25 scoring is only applied to tokenized indexes (e.g., standard, chinese analyzers)
* and not to non-tokenized indexes (e.g., keyword analyzer). This differs from Elasticsearch,
* which applies scoring to keyword indexes as well. In Doris, keyword/non-tokenized indexes
* are primarily used for exact matching without relevance scoring.
*/
// BM25 scoring for analyzed and keyword indexes. Keyword indexes without norms
// use the Lucene default encoded norm of one.
class BM25Similarity : public Similarity {
public:
BM25Similarity();
Expand Down
49 changes: 31 additions & 18 deletions be/src/storage/predicate_collector.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -382,27 +382,40 @@ Status SearchPredicateCollector::collect_from_leaf(const TSearchClause& clause,
return Status::OK();
}

if (!column.is_extracted_column()) {
// Plain fields must use the same reader for statistics and SEARCH execution.
// Variant paths retain the branch-specific binding resolution.
auto query_type = InvertedIndexQueryType::MATCH_ANY_QUERY;
if (clause_type == "EXACT") {
query_type = InvertedIndexQueryType::EQUAL_QUERY;
} else if (clause_type == "PHRASE") {
query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY;
} else if (clause_type == "ALL") {
query_type = InvertedIndexQueryType::MATCH_ALL_QUERY;
}
const auto* selected =
DORIS_TRY(select_index_meta(index_metas, column.type(), query_type, "", ""));
index_metas = {selected};
}

ClauseTypeCategory category = get_clause_type_category(clause_type);
for (const auto* index_meta : index_metas) {
if (infer_inverted_index_reader_type(column.type(), index_meta->properties()) ==
InvertedIndexReaderType::BKD) {
return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
"Index statistics collection failed: BM25 scoring does not support numeric BKD "
"search field={}",
field_name);
}
std::set<TermInfo, TermInfoComparer> term_infos;

if (category == ClauseTypeCategory::TOKENIZED) {
if (InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) {
auto analyzed_terms =
DORIS_TRY(analyze_plain_query(value, index_meta->properties()));
term_infos.insert(analyzed_terms.begin(), analyzed_terms.end());
} else {
term_infos.insert(TermInfo(value));
}
} else if (category == ClauseTypeCategory::NON_TOKENIZED) {
if (clause_type == "TERM" &&
InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) {
auto analyzed_terms =
DORIS_TRY(analyze_plain_query(value, index_meta->properties()));
term_infos.insert(analyzed_terms.begin(), analyzed_terms.end());
} else {
term_infos.insert(TermInfo(value));
}
if (!InvertedIndexAnalyzer::should_analyzer(index_meta->properties()) ||
clause_type == "MATCH" ||
(category == ClauseTypeCategory::NON_TOKENIZED && clause_type != "TERM")) {
// Keyword indexes store the whole value, including whitespace and case.
term_infos.insert(TermInfo(value));
} else {
auto analyzed_terms = DORIS_TRY(analyze_plain_query(value, index_meta->properties()));
term_infos.insert(analyzed_terms.begin(), analyzed_terms.end());
}

std::string lucene_field_name =
Expand Down
78 changes: 78 additions & 0 deletions be/test/storage/compaction/collection_statistics_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include "exprs/vexpr_context.h"
#include "exprs/vliteral.h"
#include "exprs/vmatch_predicate.h"
#include "exprs/vsearch.h"
#include "exprs/vslot_ref.h"
#include "io/fs/local_file_system.h"
#include "runtime/exec_env.h"
Expand Down Expand Up @@ -1443,4 +1444,81 @@ TEST_F(CollectionStatisticsTest, ScoringAnalyzerContextReportsWrongFamilyCompone
EXPECT_TRUE(analyzer_context_from_properties(valid).has_value());
}

namespace {
VExprSPtr make_scoring_search(const std::string& clause_type, const std::string& value) {
TSearchClause clause;
clause.__set_clause_type(clause_type);
clause.__set_field_name("content");
clause.__set_value(value);
TSearchParam param;
param.__set_root(clause);
TExprNode node;
node.__set_node_type(TExprNodeType::SEARCH_EXPR);
node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
node.__set_search_param(param);
return VSearchExpr::create_shared(node);
}
} // namespace

TEST_F(CollectionStatisticsTest, SearchKeywordClausesKeepWholeValue) {
auto schema = std::make_shared<TabletSchema>();
TabletColumn column;
column.set_unique_id(1);
column.set_name("content");
column.set_type(FieldType::OLAP_FIELD_TYPE_STRING);
schema->append_column(column);
TabletIndex index;
index._index_id = 10;
index._index_type = IndexType::INVERTED;
index._col_unique_ids.push_back(1);
schema->append_index(std::move(index));

SearchPredicateCollector collector;
for (const auto* clause_type : {"TERM", "PHRASE", "EXACT", "MATCH", "ANY", "ALL"}) {
CollectInfoMap infos;
auto expr = make_scoring_search(clause_type, "Apple banana");
ASSERT_TRUE(collector.collect(runtime_state_.get(), schema, expr, &infos).ok());
ASSERT_EQ(infos.size(), 1U);
ASSERT_EQ(infos.begin()->second.term_infos.size(), 1U);
EXPECT_EQ(infos.begin()->second.term_infos.begin()->get_single_term(), "Apple banana");
}
}

TEST_F(CollectionStatisticsTest, SearchExactSelectsKeywordStatisticsWithDualIndexes) {
auto schema = create_tablet_schema_with_inverted_index();
TabletIndex keyword;
keyword._index_id = 10;
keyword._index_type = IndexType::INVERTED;
keyword._col_unique_ids.push_back(1);
schema->append_index(std::move(keyword));
SearchPredicateCollector collector;
CollectInfoMap infos;
auto expr = make_scoring_search("EXACT", "Apple banana");
ASSERT_TRUE(collector.collect(runtime_state_.get(), schema, expr, &infos).ok());
ASSERT_EQ(infos.size(), 1U);
EXPECT_EQ(infos.begin()->second.index_meta->index_id(), 10);
ASSERT_EQ(infos.begin()->second.term_infos.size(), 1U);
EXPECT_EQ(infos.begin()->second.term_infos.begin()->get_single_term(), "Apple banana");
}

TEST_F(CollectionStatisticsTest, SearchRejectsNumericBkdScoring) {
auto schema = std::make_shared<TabletSchema>();
TabletColumn column;
column.set_unique_id(1);
column.set_name("content");
column.set_type(FieldType::OLAP_FIELD_TYPE_INT);
schema->append_column(column);
TabletIndex index;
index._index_id = 10;
index._index_type = IndexType::INVERTED;
index._col_unique_ids.push_back(1);
schema->append_index(std::move(index));
SearchPredicateCollector collector;
CollectInfoMap infos;
auto expr = make_scoring_search("EXACT", "7");
const auto status = collector.collect(runtime_state_.get(), schema, expr, &infos);
EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED);
EXPECT_TRUE(infos.empty());
}

} // namespace doris
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@
#include <vector>

#include "CLucene/index/DocRange.h"
#include "storage/index/inverted/similarity/bm25_similarity.h"

namespace doris::segment_v2::inverted_index::query_v2 {
namespace {

class MockTermDocs : public lucene::index::TermDocs {
public:
Expand Down Expand Up @@ -69,6 +71,8 @@ class MockTermDocs : public lucene::index::TermDocs {
void close() override {}
lucene::index::TermPositions* __asTermPositions() override { return nullptr; }
int32_t docFreq() override { return _doc_freq; }
int32_t getMaxBlockFreq() override { return -1; }
int32_t getMaxBlockNorm() override { return -1; }

protected:
std::vector<uint32_t> _docs;
Expand Down Expand Up @@ -155,6 +159,8 @@ class MockTermPositions : public lucene::index::TermPositions {
bool _read_done = false;
};

} // namespace

class SegmentPostingsTest : public testing::Test {};

TEST_F(SegmentPostingsTest, test_postings_positions_with_offset) {
Expand Down Expand Up @@ -185,6 +191,22 @@ TEST_F(SegmentPostingsTest, test_segment_postings_base_constructor_next_true) {
EXPECT_EQ(base.norm(), 1);
}

TEST_F(SegmentPostingsTest, KeywordPostingsUseUnitNorm) {
auto similarity = std::make_shared<segment_v2::BM25Similarity>(1.0F, 1.0F);
TermDocsPtr iter(new MockTermDocs({1, 3}, {1, 7}, {0, 0}, 2));
SegmentPostings postings(std::move(iter), true, similarity, false);
EXPECT_EQ(postings.doc(), 1);
EXPECT_EQ(postings.norm(), 1);
EXPECT_FLOAT_EQ(similarity->score(postings.freq(), postings.norm()), 1.0F);
EXPECT_GE(postings.block_max_score(), similarity->score(postings.freq(), postings.norm()));
EXPECT_EQ(postings.advance(), 3);
EXPECT_EQ(postings.norm(), 1);
EXPECT_NEAR(similarity->score(postings.freq(), postings.norm()), 7.0F * 2.2F / 8.2F, 1e-6F);
EXPECT_GE(postings.block_max_score(), similarity->score(postings.freq(), postings.norm()));
EXPECT_GE(postings.max_score(), similarity->score(postings.freq(), postings.norm()));
EXPECT_EQ(postings.max_block_norm(), 1);
}

TEST_F(SegmentPostingsTest, test_segment_postings_base_constructor_next_false) {
TermDocsPtr ptr(new MockTermDocs({}, {}, {}, 0));
SegmentPostings base(std::move(ptr), true, nullptr);
Expand Down
Loading
Loading