diff --git a/be/src/storage/compaction/collection_statistics.cpp b/be/src/storage/compaction/collection_statistics.cpp index 8f21b18e34fbe7..3aeb129d93f45b 100644 --- a/be/src/storage/compaction/collection_statistics.cpp +++ b/be/src/storage/compaction/collection_statistics.cpp @@ -209,7 +209,8 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, int3 // them, together with the field's token count, in the norms. A segment written without // norms would feed a zero avgdl, or rank its rows as zero-length documents next to the // segments that have norms, so refuse to score the collection. An index that is not - // analyzed never writes norms and is left as it is. + // analyzed never writes norms; use an average length of one when its legacy + // metadata does not include the total term frequency. const auto token_count = index_reader->sumTotalTermFreq(ws_field_name.c_str()); if (!token_count.has_value() && segment_v2::inverted_index::InvertedIndexAnalyzer::should_analyzer( @@ -220,7 +221,7 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, int3 "for a variant path, when inverted_index_skip_norms_for_variant is on", seg_path, StringHelper::to_string(ws_field_name)); } - _total_num_tokens[ws_field_name] += token_count.value_or(0); + _total_num_tokens[ws_field_name] += token_count.value_or(index_reader->maxDoc()); for (const auto& term_info : collect_info.term_infos) { auto iter = TermIterator::create(io_ctx, false, index_reader, ws_field_name, diff --git a/be/src/storage/index/inverted/query_v2/segment_postings.h b/be/src/storage/index/inverted/query_v2/segment_postings.h index 316d81d66d4e82..3d6268bb05f1ab 100644 --- a/be/src/storage/index/inverted/query_v2/segment_postings.h +++ b/be/src/storage/index/inverted/query_v2/segment_postings.h @@ -49,9 +49,11 @@ class SegmentPostings : public Postings { public: using IterVariant = std::variant; - explicit SegmentPostings(TermDocsPtr iter, bool enable_scoring, SimilarityPtr similarity) + explicit SegmentPostings(TermDocsPtr iter, bool enable_scoring, SimilarityPtr similarity, + bool has_norms = true) : _iter(std::move(iter)), _enable_scoring(enable_scoring), + _has_norms(has_norms), _similarity(std::move(similarity)) { if (auto* p = std::get_if(&_iter)) { _raw_iter = p->get(); @@ -59,9 +61,11 @@ class SegmentPostings : public Postings { _init_doc(); } - explicit SegmentPostings(TermPositionsPtr iter, bool enable_scoring, SimilarityPtr similarity) + explicit SegmentPostings(TermPositionsPtr iter, bool enable_scoring, SimilarityPtr similarity, + bool has_norms = true) : _iter(std::move(iter)), _enable_scoring(enable_scoring), + _has_norms(has_norms), _has_positions(true), _similarity(std::move(similarity)) { if (auto* p = std::get_if(&_iter)) { @@ -120,7 +124,7 @@ class SegmentPostings : public Postings { } uint32_t norm() const override { - if (!_enable_scoring || !_block.norm_many || _cursor == 0) { + if (!_enable_scoring || !_has_norms || !_block.norm_many || _cursor == 0) { return 1; } return (*_block.norm_many)[_cursor - 1]; @@ -194,24 +198,28 @@ class SegmentPostings : public Postings { return _block_max_score_cache; } int32_t max_block_freq = _raw_iter->getMaxBlockFreq(); - int32_t max_block_norm = _raw_iter->getMaxBlockNorm(); + int32_t max_block_norm = _has_norms ? _raw_iter->getMaxBlockNorm() : 1; if (max_block_freq >= 0 && max_block_norm >= 0) { _block_max_score_cache = _similarity->score(static_cast(max_block_freq), static_cast(max_block_norm)); return _block_max_score_cache; } - return _similarity->max_score(); + return max_score(); } float max_score() const { if (!_enable_scoring || !_similarity) { return std::numeric_limits::max(); } + // Missing norms have a fixed length of one, including for WAND bounds. + if (!_has_norms) { + return _similarity->score(std::numeric_limits::max(), 1); + } return _similarity->max_score(); } int32_t max_block_freq() const { return _raw_iter->getMaxBlockFreq(); } - int32_t max_block_norm() const { return _raw_iter->getMaxBlockNorm(); } + int32_t max_block_norm() const { return _has_norms ? _raw_iter->getMaxBlockNorm() : 1; } private: bool _refill() { @@ -241,6 +249,7 @@ class SegmentPostings : public Postings { lucene::index::TermDocs* _raw_iter = nullptr; uint32_t _doc = TERMINATED; bool _enable_scoring = false; + bool _has_norms = true; bool _has_positions = false; DocRange _block; @@ -253,13 +262,15 @@ class SegmentPostings : public Postings { using SegmentPostingsPtr = std::shared_ptr; inline SegmentPostingsPtr make_segment_postings(TermDocsPtr iter, bool enable_scoring, - SimilarityPtr similarity) { - return std::make_shared(std::move(iter), enable_scoring, similarity); + SimilarityPtr similarity, bool has_norms = true) { + return std::make_shared(std::move(iter), enable_scoring, similarity, + has_norms); } inline SegmentPostingsPtr make_segment_postings(TermPositionsPtr iter, bool enable_scoring, - SimilarityPtr similarity) { - return std::make_shared(std::move(iter), enable_scoring, similarity); + SimilarityPtr similarity, bool has_norms = true) { + return std::make_shared(std::move(iter), enable_scoring, similarity, + has_norms); } } // namespace doris::segment_v2::inverted_index::query_v2 \ No newline at end of file diff --git a/be/src/storage/index/inverted/query_v2/weight.h b/be/src/storage/index/inverted/query_v2/weight.h index 22d7b513030a57..4d694551549d68 100644 --- a/be/src/storage/index/inverted/query_v2/weight.h +++ b/be/src/storage/index/inverted/query_v2/weight.h @@ -151,7 +151,9 @@ class Weight { const io::IOContext* io_ctx) const { auto t = make_term_ptr(field.c_str(), term.c_str()); auto iter = make_term_doc_ptr(reader, t.get(), enable_scoring, io_ctx); - return iter ? make_segment_postings(std::move(iter), enable_scoring, similarity) : nullptr; + return iter ? make_segment_postings(std::move(iter), enable_scoring, similarity, + reader->hasNorms(field.c_str())) + : nullptr; } SegmentPostingsPtr create_position_posting(lucene::index::IndexReader* reader, @@ -168,7 +170,9 @@ class Weight { const io::IOContext* io_ctx) const { auto t = make_term_ptr(field.c_str(), term.c_str()); auto iter = make_term_positions_ptr(reader, t.get(), enable_scoring, io_ctx); - return iter ? make_segment_postings(std::move(iter), enable_scoring, similarity) : nullptr; + return iter ? make_segment_postings(std::move(iter), enable_scoring, similarity, + reader->hasNorms(field.c_str())) + : nullptr; } }; diff --git a/be/src/storage/index/inverted/similarity/bm25_similarity.h b/be/src/storage/index/inverted/similarity/bm25_similarity.h index e3bfcf1780208b..d22857962d8efd 100644 --- a/be/src/storage/index/inverted/similarity/bm25_similarity.h +++ b/be/src/storage/index/inverted/similarity/bm25_similarity.h @@ -22,14 +22,8 @@ namespace doris::segment_v2 { #include "common/compile_check_begin.h" -/** - * BM25 similarity scoring implementation for inverted index queries. - * - * Note: Currently, BM25 scoring is only applied to tokenized indexes (e.g., standard, chinese analyzers) - * and not to non-tokenized indexes (e.g., keyword analyzer). This differs from Elasticsearch, - * which applies scoring to keyword indexes as well. In Doris, keyword/non-tokenized indexes - * are primarily used for exact matching without relevance scoring. - */ +// BM25 scoring for analyzed and keyword indexes. Keyword indexes without norms +// use the Lucene default encoded norm of one. class BM25Similarity : public Similarity { public: BM25Similarity(); diff --git a/be/src/storage/predicate_collector.cpp b/be/src/storage/predicate_collector.cpp index 7ab04fa2c6ccf5..e588b1df707c23 100644 --- a/be/src/storage/predicate_collector.cpp +++ b/be/src/storage/predicate_collector.cpp @@ -382,27 +382,40 @@ Status SearchPredicateCollector::collect_from_leaf(const TSearchClause& clause, return Status::OK(); } + if (!column.is_extracted_column()) { + // Plain fields must use the same reader for statistics and SEARCH execution. + // Variant paths retain the branch-specific binding resolution. + auto query_type = InvertedIndexQueryType::MATCH_ANY_QUERY; + if (clause_type == "EXACT") { + query_type = InvertedIndexQueryType::EQUAL_QUERY; + } else if (clause_type == "PHRASE") { + query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY; + } else if (clause_type == "ALL") { + query_type = InvertedIndexQueryType::MATCH_ALL_QUERY; + } + const auto* selected = + DORIS_TRY(select_index_meta(index_metas, column.type(), query_type, "", "")); + index_metas = {selected}; + } + ClauseTypeCategory category = get_clause_type_category(clause_type); for (const auto* index_meta : index_metas) { + if (infer_inverted_index_reader_type(column.type(), index_meta->properties()) == + InvertedIndexReaderType::BKD) { + return Status::Error( + "Index statistics collection failed: BM25 scoring does not support numeric BKD " + "search field={}", + field_name); + } std::set term_infos; - - if (category == ClauseTypeCategory::TOKENIZED) { - if (InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) { - auto analyzed_terms = - DORIS_TRY(analyze_plain_query(value, index_meta->properties())); - term_infos.insert(analyzed_terms.begin(), analyzed_terms.end()); - } else { - term_infos.insert(TermInfo(value)); - } - } else if (category == ClauseTypeCategory::NON_TOKENIZED) { - if (clause_type == "TERM" && - InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) { - auto analyzed_terms = - DORIS_TRY(analyze_plain_query(value, index_meta->properties())); - term_infos.insert(analyzed_terms.begin(), analyzed_terms.end()); - } else { - term_infos.insert(TermInfo(value)); - } + if (!InvertedIndexAnalyzer::should_analyzer(index_meta->properties()) || + clause_type == "MATCH" || + (category == ClauseTypeCategory::NON_TOKENIZED && clause_type != "TERM")) { + // Keyword indexes store the whole value, including whitespace and case. + term_infos.insert(TermInfo(value)); + } else { + auto analyzed_terms = DORIS_TRY(analyze_plain_query(value, index_meta->properties())); + term_infos.insert(analyzed_terms.begin(), analyzed_terms.end()); } std::string lucene_field_name = diff --git a/be/test/storage/compaction/collection_statistics_test.cpp b/be/test/storage/compaction/collection_statistics_test.cpp index 81248beb9fbd16..617f8dfe4391bf 100644 --- a/be/test/storage/compaction/collection_statistics_test.cpp +++ b/be/test/storage/compaction/collection_statistics_test.cpp @@ -32,6 +32,7 @@ #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" #include "exprs/vmatch_predicate.h" +#include "exprs/vsearch.h" #include "exprs/vslot_ref.h" #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" @@ -1443,4 +1444,81 @@ TEST_F(CollectionStatisticsTest, ScoringAnalyzerContextReportsWrongFamilyCompone EXPECT_TRUE(analyzer_context_from_properties(valid).has_value()); } +namespace { +VExprSPtr make_scoring_search(const std::string& clause_type, const std::string& value) { + TSearchClause clause; + clause.__set_clause_type(clause_type); + clause.__set_field_name("content"); + clause.__set_value(value); + TSearchParam param; + param.__set_root(clause); + TExprNode node; + node.__set_node_type(TExprNodeType::SEARCH_EXPR); + node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN)); + node.__set_search_param(param); + return VSearchExpr::create_shared(node); +} +} // namespace + +TEST_F(CollectionStatisticsTest, SearchKeywordClausesKeepWholeValue) { + auto schema = std::make_shared(); + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + schema->append_column(column); + TabletIndex index; + index._index_id = 10; + index._index_type = IndexType::INVERTED; + index._col_unique_ids.push_back(1); + schema->append_index(std::move(index)); + + SearchPredicateCollector collector; + for (const auto* clause_type : {"TERM", "PHRASE", "EXACT", "MATCH", "ANY", "ALL"}) { + CollectInfoMap infos; + auto expr = make_scoring_search(clause_type, "Apple banana"); + ASSERT_TRUE(collector.collect(runtime_state_.get(), schema, expr, &infos).ok()); + ASSERT_EQ(infos.size(), 1U); + ASSERT_EQ(infos.begin()->second.term_infos.size(), 1U); + EXPECT_EQ(infos.begin()->second.term_infos.begin()->get_single_term(), "Apple banana"); + } +} + +TEST_F(CollectionStatisticsTest, SearchExactSelectsKeywordStatisticsWithDualIndexes) { + auto schema = create_tablet_schema_with_inverted_index(); + TabletIndex keyword; + keyword._index_id = 10; + keyword._index_type = IndexType::INVERTED; + keyword._col_unique_ids.push_back(1); + schema->append_index(std::move(keyword)); + SearchPredicateCollector collector; + CollectInfoMap infos; + auto expr = make_scoring_search("EXACT", "Apple banana"); + ASSERT_TRUE(collector.collect(runtime_state_.get(), schema, expr, &infos).ok()); + ASSERT_EQ(infos.size(), 1U); + EXPECT_EQ(infos.begin()->second.index_meta->index_id(), 10); + ASSERT_EQ(infos.begin()->second.term_infos.size(), 1U); + EXPECT_EQ(infos.begin()->second.term_infos.begin()->get_single_term(), "Apple banana"); +} + +TEST_F(CollectionStatisticsTest, SearchRejectsNumericBkdScoring) { + auto schema = std::make_shared(); + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_INT); + schema->append_column(column); + TabletIndex index; + index._index_id = 10; + index._index_type = IndexType::INVERTED; + index._col_unique_ids.push_back(1); + schema->append_index(std::move(index)); + SearchPredicateCollector collector; + CollectInfoMap infos; + auto expr = make_scoring_search("EXACT", "7"); + const auto status = collector.collect(runtime_state_.get(), schema, expr, &infos); + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + EXPECT_TRUE(infos.empty()); +} + } // namespace doris diff --git a/be/test/storage/index/inverted/query_v2/segment_postings_test.cpp b/be/test/storage/index/inverted/query_v2/segment_postings_test.cpp index 0b8e4649b3ab70..048c6d17ae34d4 100644 --- a/be/test/storage/index/inverted/query_v2/segment_postings_test.cpp +++ b/be/test/storage/index/inverted/query_v2/segment_postings_test.cpp @@ -23,8 +23,10 @@ #include #include "CLucene/index/DocRange.h" +#include "storage/index/inverted/similarity/bm25_similarity.h" namespace doris::segment_v2::inverted_index::query_v2 { +namespace { class MockTermDocs : public lucene::index::TermDocs { public: @@ -69,6 +71,8 @@ class MockTermDocs : public lucene::index::TermDocs { void close() override {} lucene::index::TermPositions* __asTermPositions() override { return nullptr; } int32_t docFreq() override { return _doc_freq; } + int32_t getMaxBlockFreq() override { return -1; } + int32_t getMaxBlockNorm() override { return -1; } protected: std::vector _docs; @@ -155,6 +159,8 @@ class MockTermPositions : public lucene::index::TermPositions { bool _read_done = false; }; +} // namespace + class SegmentPostingsTest : public testing::Test {}; TEST_F(SegmentPostingsTest, test_postings_positions_with_offset) { @@ -185,6 +191,22 @@ TEST_F(SegmentPostingsTest, test_segment_postings_base_constructor_next_true) { EXPECT_EQ(base.norm(), 1); } +TEST_F(SegmentPostingsTest, KeywordPostingsUseUnitNorm) { + auto similarity = std::make_shared(1.0F, 1.0F); + TermDocsPtr iter(new MockTermDocs({1, 3}, {1, 7}, {0, 0}, 2)); + SegmentPostings postings(std::move(iter), true, similarity, false); + EXPECT_EQ(postings.doc(), 1); + EXPECT_EQ(postings.norm(), 1); + EXPECT_FLOAT_EQ(similarity->score(postings.freq(), postings.norm()), 1.0F); + EXPECT_GE(postings.block_max_score(), similarity->score(postings.freq(), postings.norm())); + EXPECT_EQ(postings.advance(), 3); + EXPECT_EQ(postings.norm(), 1); + EXPECT_NEAR(similarity->score(postings.freq(), postings.norm()), 7.0F * 2.2F / 8.2F, 1e-6F); + EXPECT_GE(postings.block_max_score(), similarity->score(postings.freq(), postings.norm())); + EXPECT_GE(postings.max_score(), similarity->score(postings.freq(), postings.norm())); + EXPECT_EQ(postings.max_block_norm(), 1); +} + TEST_F(SegmentPostingsTest, test_segment_postings_base_constructor_next_false) { TermDocsPtr ptr(new MockTermDocs({}, {}, {}, 0)); SegmentPostings base(std::move(ptr), true, nullptr); diff --git a/regression-test/data/search/test_search_score_raw_index.out b/regression-test/data/search/test_search_score_raw_index.out new file mode 100644 index 00000000000000..19dc1db098812b --- /dev/null +++ b/regression-test/data/search/test_search_score_raw_index.out @@ -0,0 +1,136 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !V2_unscored -- +1 +3 +9 + +-- !V2_true_term -- +1 1.232144 true +3 1.232144 true +9 1.232144 true + +-- !V2_true_phrase -- +5 1.568616 true +10 1.568616 true + +-- !V2_true_exact -- +5 1.568616 true +10 1.568616 true + +-- !V2_true_or_top_two -- +5 1.568616 true +10 1.568616 true + +-- !V2_true_top_one -- +6 2.079442 true + +-- !V2_true_ascending -- +1 1.232144 true +3 1.232144 true +9 1.232144 true + +-- !V2_true_case -- +6 2.079442 true + +-- !V2_true_missing -- + +-- !V2_false_term -- +1 1.232144 true +3 1.232144 true +9 1.232144 true + +-- !V2_false_phrase -- +5 1.568616 true +10 1.568616 true + +-- !V2_false_exact -- +5 1.568616 true +10 1.568616 true + +-- !V2_false_or_top_two -- +5 1.568616 true +10 1.568616 true + +-- !V2_false_top_one -- +6 2.079442 true + +-- !V2_false_ascending -- +1 1.232144 true +3 1.232144 true +9 1.232144 true + +-- !V2_false_case -- +6 2.079442 true + +-- !V2_false_missing -- + +-- !V2_dual_exact -- +1 0.980829 true + +-- !V3_unscored -- +1 +3 +9 + +-- !V3_true_term -- +1 1.232144 true +3 1.232144 true +9 1.232144 true + +-- !V3_true_phrase -- +5 1.568616 true +10 1.568616 true + +-- !V3_true_exact -- +5 1.568616 true +10 1.568616 true + +-- !V3_true_or_top_two -- +5 1.568616 true +10 1.568616 true + +-- !V3_true_top_one -- +6 2.079442 true + +-- !V3_true_ascending -- +1 1.232144 true +3 1.232144 true +9 1.232144 true + +-- !V3_true_case -- +6 2.079442 true + +-- !V3_true_missing -- + +-- !V3_false_term -- +1 1.232144 true +3 1.232144 true +9 1.232144 true + +-- !V3_false_phrase -- +5 1.568616 true +10 1.568616 true + +-- !V3_false_exact -- +5 1.568616 true +10 1.568616 true + +-- !V3_false_or_top_two -- +5 1.568616 true +10 1.568616 true + +-- !V3_false_top_one -- +6 2.079442 true + +-- !V3_false_ascending -- +1 1.232144 true +3 1.232144 true +9 1.232144 true + +-- !V3_false_case -- +6 2.079442 true + +-- !V3_false_missing -- + +-- !V3_dual_exact -- +1 0.980829 true diff --git a/regression-test/suites/search/test_search_score_raw_index.groovy b/regression-test/suites/search/test_search_score_raw_index.groovy new file mode 100644 index 00000000000000..fb80fcf2637acd --- /dev/null +++ b/regression-test/suites/search/test_search_score_raw_index.groovy @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_search_score_raw_index", "p0") { + // Score TopN pushdown accepts one ordering key. Sort ties outside the bounded query. + def scoredQuery = { hint, table, predicate, direction, limit -> + """ + SELECT id, ROUND(bm25_score, 6), bm25_score > 0 AND bm25_score < 1000000 + FROM ( + SELECT ${hint} id, score() AS bm25_score FROM ${table} + WHERE ${predicate} + ORDER BY bm25_score ${direction} LIMIT ${limit} + ) hits + ORDER BY bm25_score ${direction}, id + """ + } + for (def storageFormat : ["V2", "V3"]) { + sql "DROP TABLE IF EXISTS test_search_score_raw_index" + sql """ + CREATE TABLE test_search_score_raw_index ( + id INT NOT NULL, + raw_index_content STRING NULL, + INDEX idx_raw_content(raw_index_content) USING INVERTED + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default:1", + "inverted_index_storage_format" = "${storageFormat}" + ) + """ + // Separate inserts exercise statistics across multiple segments. + sql """INSERT INTO test_search_score_raw_index VALUES + (1, 'apple'), (2, 'banana'), (3, 'apple'), (4, 'orange'), + (5, 'apple banana'), (6, 'Apple'), (7, NULL), (8, '') + """ + sql "sync" + sql """INSERT INTO test_search_score_raw_index VALUES + (9, 'apple'), (10, 'apple banana'), (11, 'banana') + """ + sql "sync" + + def term = "search('raw_index_content:apple', '{\"mode\":\"lucene\"}')" + def phrase = "search('raw_index_content:\"apple banana\"', '{\"mode\":\"lucene\"}')" + def exact = "search('raw_index_content:EXACT(apple banana)', '{\"mode\":\"standard\"}')" + def rankedOr = "search('raw_index_content:apple OR raw_index_content:\"apple banana\"', '{\"mode\":\"lucene\"}')" + def uniqueTop = "search('raw_index_content:apple OR raw_index_content:Apple', '{\"mode\":\"lucene\"}')" + def caseSensitive = "search('raw_index_content:Apple', '{\"mode\":\"lucene\"}')" + def missing = "search('raw_index_content:pear', '{\"mode\":\"lucene\"}')" + quickTest("${storageFormat}_unscored", """ + SELECT id FROM test_search_score_raw_index WHERE ${term} ORDER BY id + """) + for (def wand : [true, false]) { + def hint = "/*+SET_VAR(enable_inverted_index_wand_query=${wand}) */" + for (def entry : [ + ["term", term, "DESC", 100], + ["phrase", phrase, "DESC", 100], + ["exact", exact, "DESC", 100], + ["or_top_two", rankedOr, "DESC", 2], + ["top_one", uniqueTop, "DESC", 1], + ["ascending", term, "ASC", 100], + ["case", caseSensitive, "DESC", 100], + ["missing", missing, "DESC", 100]]) { + quickTest("${storageFormat}_${wand}_${entry[0]}", + scoredQuery(hint, "test_search_score_raw_index", entry[1], + entry[2], entry[3])) + } + } + sql "DROP TABLE IF EXISTS test_search_score_dual_index" + sql """ + CREATE TABLE test_search_score_dual_index ( + id INT NOT NULL, + content STRING NULL, + INDEX idx_raw(content) USING INVERTED, + INDEX idx_analyzed(content) USING INVERTED + PROPERTIES("parser"="standard", "support_phrase"="true") + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation"="tag.location.default:1", + "inverted_index_storage_format"="${storageFormat}" + ) + """ + sql """INSERT INTO test_search_score_dual_index VALUES + (1, 'apple banana'), (2, 'apple'), (3, 'apple banana orange') + """ + sql "sync" + quickTest("${storageFormat}_dual_exact", + scoredQuery("", "test_search_score_dual_index", + "search('content:EXACT(apple banana)', '{\"mode\":\"standard\"}')", + "DESC", 10)) + } +}