Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions be/src/exprs/function/search/function_search_leaf.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,9 @@ Status ScalarLeafCompiler::compile(const logical::Node& leaf, const SearchLeafCo
param.query_type = segment_v2::InvertedIndexQueryType::EQUAL_QUERY;
param.num_rows = ctx.num_rows;
param.roaring = std::make_shared<roaring::Roaring>();
// SEARCH has no row-by-row fallback, so a BKD leaf must not take the
// inverted_index_skip_threshold bypass: a bypassed leaf would fail the whole query.
param.skip_try = true;
RETURN_IF_ERROR(_iterator->read_from_index(segment_v2::IndexParam {&param}));

// Physical NULL postings are excluded by the shared SEARCH leaf mask. Never
Expand Down
4 changes: 4 additions & 0 deletions be/test/exprs/function/search/function_search_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ class RecordingIndexIterator : public segment_v2::IndexIterator {
if (i_param->query_value.get_type() == TYPE_INT) {
last_int_value = i_param->query_value.get<TYPE_INT>();
}
last_skip_try = i_param->skip_try;
if (i_param->roaring != nullptr) {
i_param->roaring->add(3);
}
Expand All @@ -196,6 +197,7 @@ class RecordingIndexIterator : public segment_v2::IndexIterator {
PrimitiveType last_query_value_type = PrimitiveType::TYPE_NULL;
bool last_bool_value = false;
Int32 last_int_value = 0;
bool last_skip_try = false;
};

class RecordingDirectInvertedIndexIterator final : public segment_v2::InvertedIndexIterator {
Expand Down Expand Up @@ -3379,6 +3381,8 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryVariantNestedIntUsesDirectIndexRead
EXPECT_EQ(InvertedIndexQueryType::EQUAL_QUERY, iterator.last_query_type);
EXPECT_EQ(TYPE_INT, iterator.last_query_value_type);
EXPECT_EQ(3, iterator.last_int_value);
// SEARCH has no row fallback, so the BKD skip-threshold bypass must be disabled.
EXPECT_TRUE(iterator.last_skip_try);
}

TEST_F(FunctionSearchTest, TestMultiPhraseQueryCase) {
Expand Down
26 changes: 26 additions & 0 deletions regression-test/data/search/test_search_bkd_skip_threshold.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !dense_variant --
7

-- !dense_column --
7

-- !dense_and_text --
4

-- !dense_or --
9

-- !sparse_variant --
2

-- !sparse_column --
2

-- !dense_topn --
7
6

-- !dense_standard_mode --
4

Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

// A numeric (BKD) leaf that matches more than inverted_index_skip_threshold percent of a
// segment used to return INVERTED_INDEX_BYPASS. SEARCH has no row-by-row path, so the
// query failed with "SEARCH cannot fall back to row execution". SEARCH must always use
// the index.
suite("test_search_bkd_skip_threshold", "p0") {
sql """ set enable_common_expr_pushdown = true """
sql """ set enable_inverted_index_query_cache = true """

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Make this regression bypass the whole-SEARCH cache while testing the leaf threshold. With enable_inverted_index_query_cache=true, mixed queries such as v.n:5 AND v.p:software are served from FunctionSearch's DSL bitmap cache on the second warmup, before ScalarLeafCompiler::read_from_index runs. The fixture has only 10 rows, so the first estimate may be half of one 128-point BKD leaf (5), which is exactly the 50% boundary and does not trigger the old strict > check. The plain k:5 case can still catch the bug, but the Variant/mixed path this test claims to cover may pass on the pre-fix code. Disable the DSL cache for the warmup or use enough rows / a lower threshold to force a pre-fix failure for each path.

sql """ set inverted_index_skip_threshold = 50 """
sql """ set default_variant_enable_typed_paths_to_sparse = false """
sql """ set default_variant_enable_doc_mode = false """
// Keep both v.n and v.p as indexed subcolumns; a fuzzy budget of 1 would push one to sparse.
sql """ set default_variant_max_subcolumns_count = 2048 """

sql "DROP TABLE IF EXISTS test_search_bkd_skip_threshold_t"
sql """
CREATE TABLE test_search_bkd_skip_threshold_t (
`id` INT NOT NULL,
`k` INT NULL,
`v` variant NULL,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Pin the Variant subcolumn budget for this fixture

The p0 pipeline enables fuzzy session variables, and SessionVariable.initFuzzyModeVariables() can leave default_variant_max_subcolumns_count=1 with nested groups disabled. This untyped VARIANT inherits that budget even after the suite disables doc mode. Both n and p occur in all ten rows, so select_variant_paths() materializes only p and sends n to sparse storage. The first search('v.n:5') then fails with SEARCH index unavailable for Variant field 'v.n', independently of the threshold fix, making this new p0 test flaky. Pin the budget to at least 2 (or 0 for unlimited) before CREATE, or declare both paths explicitly.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pinned default_variant_max_subcolumns_count = 2048 before the CREATE in afc9493. Checked locally with the global set to 1: the old version fails with “SEARCH index unavailable for Variant field ‘v.n’”, the new one passes.

INDEX idx_k (k) USING INVERTED,
INDEX idx_v (v) USING INVERTED PROPERTIES(
"parser" = "unicode", "lower_case" = "true", "support_phrase" = "true")
) ENGINE=OLAP
UNIQUE KEY(`id`)
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"enable_unique_key_merge_on_write" = "true",
"disable_auto_compaction" = "true"
)
"""

// 7 of 10 rows have n = 5 and k = 5, so a BKD lookup for 5 hits 70% of the segment.
sql """INSERT INTO test_search_bkd_skip_threshold_t VALUES
(1, 5, parse_to_variant('{"n": 5, "p": "software engineer"}')),
(2, 5, parse_to_variant('{"n": 5, "p": "sales"}')),
(3, 5, parse_to_variant('{"n": 5, "p": "software developer"}')),
(4, 5, parse_to_variant('{"n": 5, "p": "sales"}')),
(5, 5, parse_to_variant('{"n": 5, "p": "software engineer"}')),
(6, 5, parse_to_variant('{"n": 5, "p": "sales"}')),
(7, 5, parse_to_variant('{"n": 5, "p": "software tester"}')),
(8, 6, parse_to_variant('{"n": 6, "p": "software engineer"}')),
(9, 6, parse_to_variant('{"n": 6, "p": "sales"}')),
(10, 7, parse_to_variant('{"n": 7, "p": "software engineer"}'))
"""

// Run each query twice first: an early run can pass on the BKD estimate, while later runs
// read the exact hit count from the query cache, which is over the threshold.
def queries = ["v.n:5", "k:5", "v.n:5 AND v.p:software", "v.n:5 OR v.n:6", "v.n:6", "k:6"]
for (int round = 0; round < 2; round++) {
for (String q : queries) {
sql "SELECT count(*) FROM test_search_bkd_skip_threshold_t WHERE search('${q}')"
}
}

qt_dense_variant """SELECT count(*) FROM test_search_bkd_skip_threshold_t WHERE search('v.n:5')"""
qt_dense_column """SELECT count(*) FROM test_search_bkd_skip_threshold_t WHERE search('k:5')"""
qt_dense_and_text """SELECT count(*) FROM test_search_bkd_skip_threshold_t
WHERE search('v.n:5 AND v.p:software')"""
qt_dense_or """SELECT count(*) FROM test_search_bkd_skip_threshold_t WHERE search('v.n:5 OR v.n:6')"""
qt_sparse_variant """SELECT count(*) FROM test_search_bkd_skip_threshold_t WHERE search('v.n:6')"""
qt_sparse_column """SELECT count(*) FROM test_search_bkd_skip_threshold_t WHERE search('k:6')"""
qt_dense_topn """SELECT id FROM test_search_bkd_skip_threshold_t
WHERE search('v.n:5') ORDER BY id DESC LIMIT 2"""
qt_dense_standard_mode """SELECT count(*) FROM test_search_bkd_skip_threshold_t
WHERE search('v.n:5 AND v.p:software', '{"mode":"standard"}')"""
}
Loading