Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
5a8590e
[chore](be) remove the dangling NOLINTEND in core/types.h
airborne12 Sep 10, 2026
c5e5e1f
[feature](index) gram library: scheme, extractor, density solver, reg…
airborne12 Sep 10, 2026
789365d
[feature](index) gram-family analyzers and the SNII write path: docs-…
airborne12 Sep 10, 2026
6f68406
[feature](index) LIKE/REGEXP push-down through GRAM_BOOLEAN_QUERY wit…
airborne12 Sep 10, 2026
2223c91
[feature](fe) validate gram tokenizer properties, gram-family analyze…
airborne12 Sep 10, 2026
df58458
[test](regression) gram index suites: parity, recall, policy recovery…
airborne12 Sep 10, 2026
6cbfa99
[fix](be) skip analyzed MATCH queries on gram segments cut with a dif…
airborne12 Sep 11, 2026
356e0af
[fix](be) adaptive density: solve on the bytes the extractor cuts, fo…
airborne12 Sep 11, 2026
91b7021
[fix](be) fence dropped-posting dictionaries behind a required SNII f…
airborne12 Sep 11, 2026
098d41d
[test](be,regression) stop-gram suite proves a dropped entry reaches …
airborne12 Sep 11, 2026
4d251a8
[test](be,regression) pin which LIKE escape forms the index accelerat…
airborne12 Sep 11, 2026
c0d7588
[test](be,regression) drop the operand-order check, and stop the suit…
airborne12 Sep 11, 2026
f62da7d
[fix](fe) validate an omitted max_gram against the default the backen…
airborne12 Sep 11, 2026
001e556
[fix](be) gram compiler: refuse ill-formed UTF-8, and stop growing a …
airborne12 Sep 11, 2026
434df14
[fix](be) gram scheme fence covers MATCH_REGEXP and a segment with no…
airborne12 Sep 11, 2026
ab1fccb
[fix](be) record dropped postings per logical index, and charge the d…
airborne12 Sep 11, 2026
3c06bd2
[test](regression) give the inheritance suite's sparse column a needl…
airborne12 Sep 11, 2026
1eb8259
[test](regression) assert the stopped-posting inheritance phase inste…
airborne12 Sep 11, 2026
da80636
[test](regression) wait for a settled index size before comparing dro…
airborne12 Sep 12, 2026
55c5f7f
[fix](be) index declaration order no longer decides what LIKE, REGEXP…
airborne12 Sep 13, 2026
a88790d
[fix](be) gram scheme fence also covers a gram segment read under a s…
airborne12 Sep 13, 2026
801776f
[fix](be) SEARCH is never answered by a gram index, and analyzed EQUA…
airborne12 Sep 14, 2026
b6a69a3
[fix](be) numeric escapes that Boost reads differently no longer cons…
airborne12 Sep 14, 2026
0d076bf
[chore](be) correct the gram extractor benchmark notes and drop its R…
airborne12 Sep 14, 2026
16bd1d9
[test](regression) the stop-gram suite no longer changes a cluster-wi…
airborne12 Sep 14, 2026
df1d33e
[fix](be) LIKE and REGEXP hand gram index failures to the scan's fall…
airborne12 Sep 15, 2026
20a5df3
[fix](be) gram parser gives up on regex syntax the scalar engines rea…
airborne12 Sep 15, 2026
a737cc5
[test](regression) the pattern recall suite drops its engine table be…
airborne12 Sep 15, 2026
49fb7d6
[fix](be) gram parser gives up on signed repeat counts that Boost rea…
airborne12 Sep 15, 2026
566e62f
[fix](be) an index under a built-in normalizer declines gram queries …
airborne12 Sep 15, 2026
8951a98
[test](fe) gram-mode ngram tokenizers stay valid with the max_ngram_d…
airborne12 Sep 15, 2026
383d8a5
[chore](be) correct the ngram tokenizer factory comments after the ma…
airborne12 Sep 15, 2026
0975168
[fix](be) Preserve gram predicate operands and count final candidates
airborne12 Sep 28, 2026
908fcfe
[chore](be) Format SNII integration and simplify gram test setup
airborne12 Sep 28, 2026
fb488b9
[test](be) Match dynamic regex fallback expectations
airborne12 Sep 28, 2026
e2d34cb
[test](regression) Correct empty regex golden results
airborne12 Sep 28, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 103 additions & 0 deletions be/benchmark/benchmark_gram_extractor.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#pragma once

#include <benchmark/benchmark.h>

#include <cstdint>
#include <string>
#include <string_view>
#include <vector>

#include "storage/index/inverted/gram/gram_extractor.h"

namespace doris::segment_v2::gram::benchmark_detail {

// Fixed corpora make repeated builds directly comparable. Corpus 0 has many distinct
// ASCII grams, 1 represents repetitive log messages, and 2 includes UTF-8 boundaries.
inline std::string make_row(size_t size, int64_t corpus) {
std::string row;
row.reserve(size);
if (corpus == 0) {
uint64_t seed = 0x67538;
constexpr std::string_view alphabet =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 :_=/.-";
while (row.size() < size) {
seed = seed * 6364136223846793005ULL + 1442695040888963407ULL;
row.push_back(alphabet[(seed >> 32) % alphabet.size()]);
}
return row;
}
const std::string_view chunk =
corpus == 1 ? "2026-09-06 INFO rpc request completed service=Frontend latency_ms=17 "
: "2026-09-06 INFO 查询完成 service=Frontend 手机日志 latency_ms=17 ";
while (row.size() + chunk.size() <= size) {
row.append(chunk);
}
row.append(size - row.size(), 'x');
return row;
}

inline uint64_t gram_digest(const std::vector<std::string_view>& grams) {
uint64_t hash = 14695981039346656037ULL;
for (auto gram : grams) {
for (unsigned char byte : gram) {
hash = (hash ^ byte) * 1099511628211ULL;
}
hash = (hash ^ 0xFF) * 1099511628211ULL;
}
return hash;
}

// fresh=0 reuses one extractor and its output buffer across rows, as the index writers do;
// fresh=1 constructs both for every row, a synthetic measure of per-row construction and
// allocation cost. Scheme fields other than mode and lower_case keep the GramScheme defaults
// in gram_scheme.h. This measures tokenization, not SQL throughput.
inline void GramExtraction(benchmark::State& state) {
GramScheme scheme;
scheme.mode = state.range(1) == 0 ? GramMode::DENSE : GramMode::SPARSE;
scheme.lower_case = state.range(2) != 0;
const std::string row = make_row(state.range(0), state.range(3));
GramExtractor reused(scheme);
std::vector<std::string_view> grams;
reused.extract(row, &grams);
state.SetLabel("digest=" + std::to_string(gram_digest(grams)));
state.counters["grams_per_row"] = static_cast<double>(grams.size());

for ([[maybe_unused]] auto _ : state) {
if (state.range(4) != 0) {
GramExtractor fresh(scheme);
std::vector<std::string_view> fresh_grams;
fresh.extract(row, &fresh_grams);
benchmark::DoNotOptimize(fresh_grams);
benchmark::ClobberMemory();
} else {
reused.extract(row, &grams);
benchmark::DoNotOptimize(grams);
benchmark::ClobberMemory();
}
}
state.SetBytesProcessed(state.iterations() * static_cast<int64_t>(row.size()));
state.SetItemsProcessed(state.iterations());
}

BENCHMARK(GramExtraction)
->ArgsProduct({{128, 4096, 65536, 1048576}, {0, 1}, {0, 1}, {0, 1, 2}, {0, 1}})
->ArgNames({"bytes", "sparse", "lower_case", "corpus", "fresh"});

} // namespace doris::segment_v2::gram::benchmark_detail
3 changes: 2 additions & 1 deletion be/benchmark/benchmark_main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include "benchmark_column_array_view_distance.hpp"
#include "benchmark_fastunion.hpp"
#include "benchmark_fmod.hpp"
#include "benchmark_gram_extractor.hpp"
#include "benchmark_hll_merge.hpp"
#include "benchmark_hybrid_set.hpp"
#include "benchmark_json_extract.hpp"
Expand Down Expand Up @@ -92,7 +93,7 @@ static void Example1(benchmark::State& state) {
state.ResumeTiming();

// do test
for (auto _ : state) {
for ([[maybe_unused]] auto _ : state) {
auto str_col = ColumnString::create();
for (auto& v : vals) {
str_col->insert_data(v.data(), v.size());
Expand Down
52 changes: 52 additions & 0 deletions be/src/common/config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1346,6 +1346,58 @@ double get_inverted_index_candidate_pushdown_ratio() {
// inverted index match bitmap cache size
DEFINE_String(inverted_index_query_cache_limit, "10%");

namespace {

bool valid_gram_index_candidate_ratio(int32_t value) {
return value >= 0;
}

bool valid_gram_index_candidate_min_rows(int32_t value) {
return value >= 0;
}

bool valid_gram_index_min_literal_bytes(int32_t value) {
return value >= 1;
}

// A share of windows: (0, 1000]. 0 would ask for nothing and a negative value used to reach
// the writer's checked cast and fail every gram-index load until the config was repaired.
bool valid_gram_index_density_coverage_permille(int32_t value) {
return value >= 1 && value <= 1000;
}

bool valid_gram_index_density_sample_bytes(int64_t value) {
return value >= 0;
}

} // namespace

// Whether LIKE/REGEXP tries to compile a constant pattern into a gram boolean query pushed down
// to a gram-family inverted index (master switch).
DEFINE_mBool(enable_gram_index_regexp, "true");

// Cost gate for the gram boolean query: give up pruning above this share (in basis points) of a
// segment's rows, and only on segments of at least this many rows. Giving up returns the segment's
// whole docid range, so it can never drop a candidate row.
DEFINE_mInt32(gram_index_max_candidate_ratio_bp, "15");
DEFINE_Validator(gram_index_max_candidate_ratio_bp, valid_gram_index_candidate_ratio);
DEFINE_mInt32(gram_index_candidate_ratio_min_rows, "65536");

// On by default: a configured density is wrong on every column but the one it was chosen for,
// and the solve costs one pass over a bounded sample.
DEFINE_mBool(enable_gram_index_adaptive_density, "true");
// 12 bytes at 95% of windows. Both are the promise rather than a tuning pair -- what a user
// may reasonably change is how short a literal they expect to find, not the boundary rate.
// A literal shorter than the scheme's max_gram cannot hold a whole gram, so the writer
// raises the promise to max_gram where a tokenizer sets a longer one.
DEFINE_mInt32(gram_index_min_literal_bytes, "12");
DEFINE_mInt32(gram_index_density_coverage_permille, "950");
Comment thread
airborne12 marked this conversation as resolved.
Resolved
DEFINE_mInt64(gram_index_density_sample_bytes, "4194304");
DEFINE_Validator(gram_index_candidate_ratio_min_rows, valid_gram_index_candidate_min_rows);
DEFINE_Validator(gram_index_min_literal_bytes, valid_gram_index_min_literal_bytes);
DEFINE_Validator(gram_index_density_coverage_permille, valid_gram_index_density_coverage_permille);
DEFINE_Validator(gram_index_density_sample_bytes, valid_gram_index_density_sample_bytes);

// condition cache limit
DEFINE_Int16(condition_cache_limit, "512");

Expand Down
54 changes: 54 additions & 0 deletions be/src/common/config.h
Original file line number Diff line number Diff line change
Expand Up @@ -1391,6 +1391,60 @@ DECLARE_String(inverted_index_query_cache_limit);
// it affects only writers created after the transition and never changes query/cache semantics.
// Release-calibrated query-planner coefficients. Both remain mutable for controlled recalibration.

// Whether LIKE/REGEXP tries to compile a constant pattern into a gram boolean query pushed down
// to a gram-family inverted index (master switch). Turning it off behaves as if the index did
// not exist -- it only gives up the speedup, it never changes query results.
//
// A pattern is compiled against the gram scheme the segment itself carries, read back from the
// core metadata of the physical index the query is about to read, so the scheme can never
// disagree with the one the writer used -- a policy that was dropped and recreated with different
// properties does not affect segments already written.
DECLARE_mBool(enable_gram_index_regexp);

// Cost gate for the gram boolean query, in basis points (1/10000) of a segment's rows: give up
// pruning once the candidate set is larger than that share. 0 disables the gate, 10000 or more can
// never fire. This is a COST switch, never a semantic one -- a gram index only produces a superset
// of the candidate rows and the expression above re-verifies each of them, so giving up means the
// query returns the whole docid range of the segment and prunes nothing. It can therefore only
// cost speed, never correctness, whatever value it is set to.
//
// The unit is basis points rather than percent because the useful range is far below one percent.
// Pruning only pays when the surviving candidates are sparse enough that whole pages of the column
// can be skipped: with B rows per remote read unit and candidates spread uniformly, the fraction of
// units still touched is 1 - (1 - C/N)^B, which is already ~1 by C/N = 1%. Measured on a 34.46M row
// httplogs table against remote object storage, cold-read speedup by candidate count was 14.1x at
// 2 candidates, 5.5x at 128, 1.5x at 25K, 1.2x at 46K, and turned into a regression from 56K
// upwards, bottoming out at 0.70x. 15 bp (0.15%) is the largest share that admitted no regression
// in that sweep while keeping every double-digit win.
DECLARE_mInt32(gram_index_max_candidate_ratio_bp);

// Row floor below which the candidate ratio gate above is not applied at all. A small segment's
// entire gram index is a few KB and one or two requests, so giving up there saves nothing
// measurable while throwing away the pruning the index really does deliver; the ratio only starts
// to mean something at a size where the skipped index IO can outweigh the rows it stops
// eliminating. The default is one Roaring container's worth of rows (65536). 0 applies the ratio
// at every segment size.
DECLARE_mInt32(gram_index_candidate_ratio_min_rows);

// Adaptive gram density: solve the boundary rate from each segment's own bytes instead of
// taking the rate configured on the tokenizer.
//
// A configured rate cannot mean the same thing on two columns. Measured at a nominal 0.25 the
// realised grams per byte were 0.204 on log text, 0.287 on URL paths and 0.182 on agent
// traces, and the coverage it bought ranged from 91.7% of 12-byte literals to 98.9% -- under
// serving one column while overpaying on another, in every posting list. Solving instead
// makes the promise the constant and the rate the variable.
DECLARE_mBool(enable_gram_index_adaptive_density);
// The promise the solve keeps, not a tuning pair: literals of at least this many bytes are
// findable, for this share of the column's own windows of that length. Both are dimensionless
// and the same on every dataset; what varies is the density they resolve to.
DECLARE_mInt32(gram_index_min_literal_bytes);
DECLARE_mInt32(gram_index_density_coverage_permille);
// How much of a segment is held back to solve on. The sample is buffered rather than
// tokenized, so this is a transient memory cost and a bound on how long the write path waits
// before it can cut anything; the histogram behind the solve is a fixed 256 KB regardless.
DECLARE_mInt64(gram_index_density_sample_bytes);

// condition cache limit
DECLARE_Int16(condition_cache_limit);

Expand Down
1 change: 0 additions & 1 deletion be/src/core/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -573,7 +573,6 @@ struct NativeType<Decimal256> {
using Type = wide::Int256;
};

// NOLINTEND(readability-function-size)
} // namespace doris

/// Specialization of `std::hash` for the Decimal<T> types.
Expand Down
6 changes: 6 additions & 0 deletions be/src/exec/operator/olap_scan_operator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,12 @@ Status OlapScanLocalState::_init_profile() {
ADD_COUNTER_WITH_LEVEL(_segment_profile, "InvertedIndexDowngradeCount", TUnit::UNIT, 1);
_inverted_index_conjuncts_short_circuited_counter = ADD_COUNTER_WITH_LEVEL(
_segment_profile, "InvertedIndexConjunctsShortCircuited", TUnit::UNIT, 1);
_gram_index_filter_counter =
ADD_COUNTER_WITH_LEVEL(_segment_profile, "RowsGramIndexFiltered", TUnit::UNIT, 1);
_gram_index_candidate_counter =
ADD_COUNTER_WITH_LEVEL(_segment_profile, "GramIndexCandidateRows", TUnit::UNIT, 1);
_gram_index_gate_gave_up_counter =
ADD_COUNTER_WITH_LEVEL(_segment_profile, "GramIndexGateGaveUp", TUnit::UNIT, 1);
_inverted_index_analyzer_timer =
ADD_TIMER_WITH_LEVEL(_segment_profile, "InvertedIndexAnalyzerTime", 1);
_inverted_index_lookup_timer =
Expand Down
4 changes: 4 additions & 0 deletions be/src/exec/operator/olap_scan_operator.h
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,10 @@ class OlapScanLocalState final : public ScanLocalState<OlapScanLocalState> {
RuntimeProfile::Counter* _inverted_index_searcher_cache_miss_counter = nullptr;
RuntimeProfile::Counter* _inverted_index_downgrade_count_counter = nullptr;
RuntimeProfile::Counter* _inverted_index_conjuncts_short_circuited_counter = nullptr;
// Approximate (gram) index: rows pruned / candidate rows left for expression re-verification.
RuntimeProfile::Counter* _gram_index_filter_counter = nullptr;
RuntimeProfile::Counter* _gram_index_candidate_counter = nullptr;
RuntimeProfile::Counter* _gram_index_gate_gave_up_counter = nullptr;
RuntimeProfile::Counter* _inverted_index_analyzer_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_lookup_timer = nullptr;

Expand Down
3 changes: 3 additions & 0 deletions be/src/exec/scan/olap_scanner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -877,6 +877,9 @@ void OlapScanner::_collect_profile_before_close() {
stats.inverted_index_downgrade_count);
COUNTER_UPDATE(local_state->_inverted_index_conjuncts_short_circuited_counter,
stats.inverted_index_conjuncts_short_circuited);
COUNTER_UPDATE(local_state->_gram_index_filter_counter, stats.rows_gram_index_filtered);
COUNTER_UPDATE(local_state->_gram_index_candidate_counter, stats.gram_index_candidate_rows);
COUNTER_UPDATE(local_state->_gram_index_gate_gave_up_counter, stats.gram_index_gate_gave_up);
COUNTER_UPDATE(local_state->_inverted_index_analyzer_timer,
stats.inverted_index_analyzer_timer);
COUNTER_UPDATE(local_state->_inverted_index_lookup_timer, stats.inverted_index_lookup_timer);
Expand Down
13 changes: 13 additions & 0 deletions be/src/exprs/function/function.h
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,15 @@ class IFunctionBase {
}
}

// True when this function's index push-down can only ever answer with an approximate
// (superset) candidate set instead of the exact row set. Such a result is usable in one
// place only -- narrowing the candidate rows of a conjunct that stays pushed down for
// row-level re-verification -- so VExpr skips the push-down entirely wherever it would be
// computed and then dropped, rather than paying for the index reads first.
virtual bool index_result_is_approximate() const { return false; }

// VExpr calls this after binding the indexed fields to storage-compatible types and
// iterators; the arguments carry the call's literals, in child order.
virtual Status evaluate_inverted_index(
const ColumnsWithTypeAndName& arguments,
const std::vector<IndexFieldNameAndTypePair>& data_type_with_names,
Expand Down Expand Up @@ -507,6 +516,10 @@ class DefaultFunction final : public IFunctionBase {
return function->close(context, scope);
}

bool index_result_is_approximate() const override {
return function->index_result_is_approximate();
}

Status evaluate_inverted_index(
const ColumnsWithTypeAndName& args,
const std::vector<IndexFieldNameAndTypePair>& data_type_with_names,
Expand Down
Loading
Loading