From 5a8590e6dc278cc24de036c4dc74c073ac23fbbb Mon Sep 17 00:00:00 2001 From: airborne12 Date: Thu, 10 Sep 2026 20:55:19 +0800 Subject: [PATCH 01/36] [chore](be) remove the dangling NOLINTEND in core/types.h The file closes a NOLINT region it never opened; clang-tidy on any change nearby reports it. Kept as its own commit so it can be dropped alone. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01METuP3aVRn8dfCU62PivnF --- be/src/core/types.h | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/core/types.h b/be/src/core/types.h index e732c31d3e8f1e..137dac8214276e 100644 --- a/be/src/core/types.h +++ b/be/src/core/types.h @@ -573,7 +573,6 @@ struct NativeType { using Type = wide::Int256; }; -// NOLINTEND(readability-function-size) } // namespace doris /// Specialization of `std::hash` for the Decimal types. From c5e5e1fcc4d971c69e70d1585e37a6c6ff691ad0 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Thu, 10 Sep 2026 20:55:19 +0800 Subject: [PATCH 02/36] [feature](index) gram library: scheme, extractor, density solver, regex AST and gram compiler The pieces that turn a LIKE/REGEXP pattern into the set of grams every matching row must contain, independent of any index format: - GramScheme: the parameters of a gram-family tokenizer (dense sliding window or sparse content-defined grams, min/max length, density, case folding), with the on-disk encoding rules. - GramExtractor: cuts a row into grams. ASCII runs are cut by the scheme; non-ASCII text is not indexed and separates the runs around it, on the write side and the query side alike, so the non-ASCII part of a literal adds no constraint and a wholly non-ASCII pattern runs as a scan. - DensitySolver: solves a segment's boundary density from its own bytes so that 95% of all 12-byte windows still contain a whole gram, in one pass over a bounded sample with a fixed 256 KB histogram. - A RE2-subset regex parser and RegexGramCompiler (Cox 2012): derives the grams a match must contain for REGEXP and LIKE, degrading to ALL wherever the derivation cannot be made reliably (lookbehind, oversized classes, ill-formed UTF-8, too many leaves). - gram_family: resolves whether an index's analyzer is gram family. Tests: unit tests per component, a differential fuzz test against RE2, a no-false-negative property test over a corpus with ill-formed UTF-8, NUL bytes and 4 KiB rows across a scheme matrix, recall and literal-coverage tests, the density solver's corpus calibration, and the df-distribution and section-size measurements that motivated the write-path policies. A micro-benchmark for the extractor lives under be/benchmark. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01METuP3aVRn8dfCU62PivnF --- be/benchmark/README_gram_extractor.md | 55 + be/benchmark/benchmark_gram_extractor.hpp | 101 ++ be/benchmark/benchmark_main.cpp | 3 +- .../index/inverted/gram/gram_density.cpp | 125 +++ .../index/inverted/gram/gram_density.h | 87 ++ .../index/inverted/gram/gram_extractor.cpp | 275 +++++ .../index/inverted/gram/gram_extractor.h | 106 ++ .../index/inverted/gram/gram_family.cpp | 50 + .../storage/index/inverted/gram/gram_family.h | 54 + .../index/inverted/gram/gram_query.cpp | 289 +++++ .../storage/index/inverted/gram/gram_query.h | 71 ++ .../index/inverted/gram/gram_scheme.cpp | 113 ++ .../storage/index/inverted/gram/gram_scheme.h | 62 ++ .../storage/index/inverted/gram/regex_ast.cpp | 800 ++++++++++++++ .../storage/index/inverted/gram/regex_ast.h | 69 ++ .../inverted/gram/regex_gram_compiler.cpp | 720 +++++++++++++ .../index/inverted/gram/regex_gram_compiler.h | 98 ++ .../gram/gram_density_corpus_test.cpp | 126 +++ .../index/inverted/gram/gram_density_test.cpp | 168 +++ .../gram/gram_df_distribution_test.cpp | 148 +++ .../inverted/gram/gram_extractor_test.cpp | 289 +++++ .../gram/gram_literal_coverage_test.cpp | 204 ++++ .../index/inverted/gram/gram_query_test.cpp | 77 ++ .../index/inverted/gram/gram_scheme_test.cpp | 106 ++ .../index/inverted/gram/regex_ast_test.cpp | 175 ++++ .../gram/regex_gram_compiler_test.cpp | 159 +++ .../inverted/gram/regex_gram_fuzz_test.cpp | 179 ++++ .../regex_gram_no_false_negative_test.cpp | 990 ++++++++++++++++++ .../inverted/gram/regex_gram_recall_test.cpp | 846 +++++++++++++++ 29 files changed, 6544 insertions(+), 1 deletion(-) create mode 100644 be/benchmark/README_gram_extractor.md create mode 100644 be/benchmark/benchmark_gram_extractor.hpp create mode 100644 be/src/storage/index/inverted/gram/gram_density.cpp create mode 100644 be/src/storage/index/inverted/gram/gram_density.h create mode 100644 be/src/storage/index/inverted/gram/gram_extractor.cpp create mode 100644 be/src/storage/index/inverted/gram/gram_extractor.h create mode 100644 be/src/storage/index/inverted/gram/gram_family.cpp create mode 100644 be/src/storage/index/inverted/gram/gram_family.h create mode 100644 be/src/storage/index/inverted/gram/gram_query.cpp create mode 100644 be/src/storage/index/inverted/gram/gram_query.h create mode 100644 be/src/storage/index/inverted/gram/gram_scheme.cpp create mode 100644 be/src/storage/index/inverted/gram/gram_scheme.h create mode 100644 be/src/storage/index/inverted/gram/regex_ast.cpp create mode 100644 be/src/storage/index/inverted/gram/regex_ast.h create mode 100644 be/src/storage/index/inverted/gram/regex_gram_compiler.cpp create mode 100644 be/src/storage/index/inverted/gram/regex_gram_compiler.h create mode 100644 be/test/storage/index/inverted/gram/gram_density_corpus_test.cpp create mode 100644 be/test/storage/index/inverted/gram/gram_density_test.cpp create mode 100644 be/test/storage/index/inverted/gram/gram_df_distribution_test.cpp create mode 100644 be/test/storage/index/inverted/gram/gram_extractor_test.cpp create mode 100644 be/test/storage/index/inverted/gram/gram_literal_coverage_test.cpp create mode 100644 be/test/storage/index/inverted/gram/gram_query_test.cpp create mode 100644 be/test/storage/index/inverted/gram/gram_scheme_test.cpp create mode 100644 be/test/storage/index/inverted/gram/regex_ast_test.cpp create mode 100644 be/test/storage/index/inverted/gram/regex_gram_compiler_test.cpp create mode 100644 be/test/storage/index/inverted/gram/regex_gram_fuzz_test.cpp create mode 100644 be/test/storage/index/inverted/gram/regex_gram_no_false_negative_test.cpp create mode 100644 be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp diff --git a/be/benchmark/README_gram_extractor.md b/be/benchmark/README_gram_extractor.md new file mode 100644 index 00000000000000..d510eed7c99347 --- /dev/null +++ b/be/benchmark/README_gram_extractor.md @@ -0,0 +1,55 @@ +# Gram extraction benchmark + +`GramExtraction` measures the real `GramExtractor::extract` implementation, including +ASCII folding, boundary selection, per-row deduplication and allocations. It is not a +benchmark of complete token streams, index construction, or SQL queries. + +## Build and run + +Set `BUILD_TYPE=RELEASE` in the local `custom_env.sh` for this performance build, then +use the standard build entry point: + +```bash +./build.sh --benchmark -j "$(nproc)" +./output/be/lib/benchmark_test \ + --benchmark_filter='GramExtraction/' \ + --benchmark_min_time=0.3s \ + --benchmark_repetitions=7 \ + --benchmark_out=gram-extraction.json \ + --benchmark_out_format=json +``` + +On Linux, make the build's `libjvm.so` discoverable through `LD_LIBRARY_PATH` if +needed. Keep correctness tests in the normal ASAN build; do not compare an ASAN +timing with a RELEASE timing. + +## Matrix + +| Argument | Values | +| --- | --- | +| `bytes` | 128, 4096, 65536, 1048576 | +| `sparse` | 0: dense; 1: sparse | +| `lower_case` | 0: preserve ASCII case; 1: fold ASCII case | +| `corpus` | 0: deterministic diverse ASCII; 1: repeated ASCII log; 2: mixed UTF-8 log | +| `fresh` | 1: construct/destroy extractor and output vector per row; 0: reuse both | + +Other parameters use `GramScheme` defaults: minimum length 3, maximum length 16, +density 250/1000, hash version 1. All inputs are generated before timing and reused +as hot input. The `fresh=1` lifetime matches per-value gram tokenization in the +writer; `fresh=0` isolates extraction after constructor and buffer reuse costs. + +`grams_per_row` and `digest` record the initial extraction outside timing. Compare +them before interpreting timing differences between implementations. Correctness +of repeated calls is covered by the gram unit tests, not by a timing threshold. + +For comparisons, use identical compiler flags, dependencies, input and CPU affinity. +Do not compile or run regression workloads concurrently with timing. Run both orders +(for example fixed/old/old/fixed), retain the per-repetition CPU `ns/op`, and inspect +dispersion as well as medians. Extend `benchmark_min_time` for cases with few +iterations or high variability. CPU frequency scaling, SMT and unrelated host load +can still affect measurements. + +Report `fresh` and `reused` separately, as well as corpus and size: constructor +savings on short rows can conceal an extraction-only cost on long rows. A throughput +ratio from this single-threaded, hot-input microbenchmark is not an end-to-end +database speedup or a statement about all tokenizer parameter combinations. diff --git a/be/benchmark/benchmark_gram_extractor.hpp b/be/benchmark/benchmark_gram_extractor.hpp new file mode 100644 index 00000000000000..5d476a5003071a --- /dev/null +++ b/be/benchmark/benchmark_gram_extractor.hpp @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "storage/index/inverted/gram/gram_extractor.h" + +namespace doris::segment_v2::gram::benchmark_detail { + +// Fixed corpora make repeated builds directly comparable. Corpus 0 has many distinct +// ASCII grams, 1 represents repetitive log messages, and 2 includes UTF-8 boundaries. +inline std::string make_row(size_t size, int64_t corpus) { + std::string row; + row.reserve(size); + if (corpus == 0) { + uint64_t seed = 0x67538; + constexpr std::string_view alphabet = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 :_=/.-"; + while (row.size() < size) { + seed = seed * 6364136223846793005ULL + 1442695040888963407ULL; + row.push_back(alphabet[(seed >> 32) % alphabet.size()]); + } + return row; + } + const std::string_view chunk = + corpus == 1 ? "2026-09-06 INFO rpc request completed service=Frontend latency_ms=17 " + : "2026-09-06 INFO 查询完成 service=Frontend 手机日志 latency_ms=17 "; + while (row.size() + chunk.size() <= size) { + row.append(chunk); + } + row.append(size - row.size(), 'x'); + return row; +} + +inline uint64_t gram_digest(const std::vector& grams) { + uint64_t hash = 14695981039346656037ULL; + for (auto gram : grams) { + for (unsigned char byte : gram) { + hash = (hash ^ byte) * 1099511628211ULL; + } + hash = (hash ^ 0xFF) * 1099511628211ULL; + } + return hash; +} + +// fresh=1 follows the writer's per-value tokenizer lifetime; fresh=0 isolates +// extraction with reusable buffers. This measures tokenization, not SQL throughput. +inline void GramExtraction(benchmark::State& state) { + GramScheme scheme; + scheme.mode = state.range(1) == 0 ? GramMode::DENSE : GramMode::SPARSE; + scheme.lower_case = state.range(2) != 0; + const std::string row = make_row(state.range(0), state.range(3)); + GramExtractor reused(scheme); + std::vector grams; + reused.extract(row, &grams); + state.SetLabel("digest=" + std::to_string(gram_digest(grams))); + state.counters["grams_per_row"] = static_cast(grams.size()); + + for ([[maybe_unused]] auto _ : state) { + if (state.range(4) != 0) { + GramExtractor fresh(scheme); + std::vector fresh_grams; + fresh.extract(row, &fresh_grams); + benchmark::DoNotOptimize(fresh_grams); + benchmark::ClobberMemory(); + } else { + reused.extract(row, &grams); + benchmark::DoNotOptimize(grams); + benchmark::ClobberMemory(); + } + } + state.SetBytesProcessed(state.iterations() * static_cast(row.size())); + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(GramExtraction) + ->ArgsProduct({{128, 4096, 65536, 1048576}, {0, 1}, {0, 1}, {0, 1, 2}, {0, 1}}) + ->ArgNames({"bytes", "sparse", "lower_case", "corpus", "fresh"}); + +} // namespace doris::segment_v2::gram::benchmark_detail diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp index d68509f785b23e..4eb676e50307a7 100644 --- a/be/benchmark/benchmark_main.cpp +++ b/be/benchmark/benchmark_main.cpp @@ -31,6 +31,7 @@ #include "benchmark_column_array_view_distance.hpp" #include "benchmark_fastunion.hpp" #include "benchmark_fmod.hpp" +#include "benchmark_gram_extractor.hpp" #include "benchmark_hll_merge.hpp" #include "benchmark_hybrid_set.hpp" #include "benchmark_json_extract.hpp" @@ -92,7 +93,7 @@ static void Example1(benchmark::State& state) { state.ResumeTiming(); // do test - for (auto _ : state) { + for ([[maybe_unused]] auto _ : state) { auto str_col = ColumnString::create(); for (auto& v : vals) { str_col->insert_data(v.data(), v.size()); diff --git a/be/src/storage/index/inverted/gram/gram_density.cpp b/be/src/storage/index/inverted/gram/gram_density.cpp new file mode 100644 index 00000000000000..28627bdb46e938 --- /dev/null +++ b/be/src/storage/index/inverted/gram/gram_density.cpp @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/gram/gram_density.h" + +#include + +#include "storage/index/inverted/gram/gram_extractor.h" + +namespace doris::segment_v2::gram { + +DensitySolver::DensitySolver(size_t min_literal_len, size_t max_gram_len) + : _min_literal_len(min_literal_len), + _max_gram_len(max_gram_len), + _histogram(kHashValues, 0) {} + +void DensitySolver::observe(std::string_view value) { + if (_min_literal_len < _max_gram_len || _max_gram_len < 2) { + return; + } + // Boundaries with a whole gram's room after them inside a window: a boundary at k yields + // [k, k+max_gram_len) at worst, so k may not exceed len - max_gram_len. That leaves + // `per_window` candidate positions in a window of the promised length. + const size_t per_window = _min_literal_len - _max_gram_len + 1; + + size_t i = 0; + const size_t n = value.size(); + while (i < n) { + // Only ASCII runs: they are the only bytes the extractor indexes, so a window + // spanning anything else could never produce a gram. + if (static_cast(value[i]) >= 0x80) { + ++i; + continue; + } + size_t j = i; + while (j < n && static_cast(value[j]) < 0x80) { + ++j; + } + const std::string_view run = value.substr(i, j - i); + i = j; + if (run.size() < _min_literal_len) { + continue; + } + // Sliding minimum over the run's pair hashes, one window per start position. The + // monotonic queue keeps this linear in the run rather than quadratic in the window, + // which is what makes it affordable on the write path. + const size_t pairs = run.size() - 1; + _mono.clear(); + size_t head = 0; + for (size_t p = 0; p < pairs; ++p) { + const uint16_t h = + boundary_hash16(static_cast(run[p]), static_cast(run[p + 1])); + while (_mono.size() > head && + boundary_hash16(static_cast(run[_mono.back()]), + static_cast(run[_mono.back() + 1])) >= h) { + _mono.pop_back(); + } + _mono.push_back(static_cast(p)); + if (p + 1 < per_window) { + continue; + } + const size_t window_start = p + 1 - per_window; + while (_mono[head] < window_start) { + ++head; + } + const uint16_t min_hash = boundary_hash16(static_cast(run[_mono[head]]), + static_cast(run[_mono[head] + 1])); + ++_histogram[min_hash]; + ++_windows; + } + } +} + +uint16_t DensitySolver::solve(uint32_t coverage_permille) const { + if (_windows == 0 || coverage_permille == 0) { + // No evidence, or nothing asked for. Stay where a configured default would have been + // rather than inventing a rate from nothing. + return kMaxSolvedDensityPermille; + } + // A window is covered exactly when its minimum is below the threshold, so the prefix sum + // of the histogram IS the coverage curve: walk it until the requested share is reached and + // the threshold that achieves it -- and nothing larger -- is one past that hash. + const uint64_t need = (static_cast(coverage_permille) * _windows + 999) / 1000; + uint64_t seen = 0; + uint32_t threshold = kHashValues; + for (size_t h = 0; h < kHashValues; ++h) { + seen += _histogram[h]; + if (seen >= need) { + // `is_boundary` compares strictly, so the threshold has to clear this hash. + threshold = static_cast(h) + 1; + break; + } + } + // GramExtractor derives its threshold as density_permille * 65536 / 1000 with integer + // division; round up or the resolved density can land a notch below what was solved for. + uint32_t permille = + static_cast((static_cast(threshold) * 1000 + 65535) / 65536); + permille = std::clamp(permille, kMinSolvedDensityPermille, kMaxSolvedDensityPermille); + return static_cast(permille); +} + +uint16_t solve_density_permille(const std::vector& sample_rows, size_t min_literal_len, + size_t max_gram_len, uint32_t coverage_permille) { + DensitySolver solver(min_literal_len, max_gram_len); + for (const std::string& row : sample_rows) { + solver.observe(row); + } + return solver.solve(coverage_permille); +} + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/gram_density.h b/be/src/storage/index/inverted/gram/gram_density.h new file mode 100644 index 00000000000000..64a0dd950c8814 --- /dev/null +++ b/be/src/storage/index/inverted/gram/gram_density.h @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace doris::segment_v2::gram { + +// Bounds on a solved density, in permille. They exist so a pathological sample cannot produce +// an index that holds nothing or one that holds a gram at every position; they are not a +// tuning range, and a solve landing on either edge means the sample was degenerate. +inline constexpr uint16_t kMinSolvedDensityPermille = 5; +inline constexpr uint16_t kMaxSolvedDensityPermille = 500; + +// Accumulates the evidence a density is solved from, one row at a time. +// +// The promise a sparse gram index makes is "a literal of at least `min_literal_len` bytes can +// be found". A window of that many bytes is indexable exactly when it holds a boundary with +// room for a whole gram after it, and a boundary is decided by the identity of a byte pair -- +// so the smallest pair hash in a window is the density at which that window starts being +// indexable. The sparsest density keeping the promise for a share q of windows is therefore +// the q-quantile of those minima, and a quantile needs no sample kept: the hashes are 16 bits, +// so a histogram over their whole range answers it exactly in one prefix scan. +// +// That matters on the write path, which is where this runs. Observing a row is linear in its +// length with a monotonic deque, the histogram is a fixed 256 KB whatever the sample size, and +// solving is one scan of 65,536 counters. Nothing is retained per row and nothing is sorted. +class DensitySolver { +public: + // `max_gram_len` is the scheme's max_gram: a boundary yields a gram only if a whole one + // fits after it, so the last max_gram_len-1 positions of a window cannot carry coverage. + // Counting them made an earlier version optimistic -- it asked for 95% of 12-byte literals + // on URL paths and delivered 86%. + DensitySolver(size_t min_literal_len, size_t max_gram_len); + + // Feeds one column value. Only ASCII runs contribute, since they are the only bytes the + // extractor indexes. + void observe(std::string_view value); + + // Windows seen so far. Zero means nothing of the promised length was present and solve() + // has no evidence to work from. + uint64_t observed_windows() const { return _windows; } + + // The sparsest density keeping the promise for `coverage_permille` of the observed + // windows, clamped to the bounds above. With no evidence it returns the ceiling rather + // than inventing a rate. + uint16_t solve(uint32_t coverage_permille) const; + + // Fixed regardless of how much has been observed. + static constexpr size_t heap_bytes() { return kHashValues * sizeof(uint32_t); } + +private: + static constexpr size_t kHashValues = 1U << 16U; + + size_t _min_literal_len; + size_t _max_gram_len; + uint64_t _windows = 0; + // Counts of per-window minimum hashes, indexed by the hash itself. + std::vector _histogram; + // Scratch reused across rows so observing a row allocates nothing. + mutable std::vector _mono; +}; + +// One-shot convenience over DensitySolver, for tests and offline analysis. +uint16_t solve_density_permille(const std::vector& sample_rows, size_t min_literal_len, + size_t max_gram_len, uint32_t coverage_permille); + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/gram_extractor.cpp b/be/src/storage/index/inverted/gram/gram_extractor.cpp new file mode 100644 index 00000000000000..23577aa648a4bb --- /dev/null +++ b/be/src/storage/index/inverted/gram/gram_extractor.cpp @@ -0,0 +1,275 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/gram/gram_extractor.h" + +#include +#include + +#include "common/logging.h" + +namespace doris::segment_v2::gram { + +// The BE storage target enables CMake unity builds (several .cpp files are compiled together, +// see UNITY_BUILD_BATCH_SIZE in be/src/storage/CMakeLists.txt), so every anonymous namespace in +// a batch is merged into one translation unit. A bare anonymous namespace then redefines any +// symbol whose name another file of the same batch happens to reuse (even in a different .cpp), +// and the batching changes as files are added to or removed from the directory, so "this batch +// only holds these files" cannot be assumed for long. Hence the extra named namespace private +// to this file, which isolates this file's anonymous namespace; the symbols inside it still +// have internal linkage (anonymous-namespace semantics are unaffected by a named enclosing +// namespace). +namespace gram_extractor_detail { + +namespace { + +// The splitmix64 finalizer, used as the boundary hash mixing function for a byte pair (a,b); it +// must match mix64 in the prototype tools/regex-ngram-model/ngram_model.cpp bit for bit, because +// the golden data was generated by that prototype. +inline uint64_t mix64(uint64_t x) { + x ^= x >> 30; + x *= 0xbf58476d1ce4e5b9ULL; + x ^= x >> 27; + x *= 0x94d049bb133111ebULL; + x ^= x >> 31; + return x; +} + +// Byte length (1/2/3/4) of the UTF-8 sequence starting with lead byte c; an illegal lead byte +// counts as 1. +inline int utf8_len(unsigned char c) { + if (c < 0x80) { + return 1; + } + if ((c >> 5) == 0x6) { + return 2; + } + if ((c >> 4) == 0xE) { + return 3; + } + if ((c >> 3) == 0x1E) { + return 4; + } + return 1; +} + +// Byte length of one valid UTF-8 code point starting at p; a truncated sequence or an invalid +// continuation byte (not 10xxxxxx) counts as illegal and returns 1 (handled as a single byte, +// emitted as a 1-gram). +inline size_t codepoint_len(const char* p, size_t remain) { + int l = utf8_len((unsigned char)p[0]); + if (l == 1 || (size_t)l > remain) { + return 1; + } + for (int k = 1; k < l; k++) { + if (((unsigned char)p[k] & 0xC0) != 0x80) { + return 1; + } + } + return l; +} + +} // namespace + +} // namespace gram_extractor_detail + +GramExtractor::GramExtractor(const GramScheme& scheme) + : _scheme(scheme), + _boundary_threshold(static_cast(scheme.density_permille) * 65536 / 1000) {} + +void GramExtractor::set_density_permille(uint16_t density_permille) { + _scheme.density_permille = density_permille; + _boundary_threshold = static_cast(density_permille) * 65536 / 1000; +} + +uint16_t boundary_hash16(uint8_t a, uint8_t b) { + const uint64_t key = ((static_cast(a) << 8) | b) ^ 0x5bd1e995ULL; + return static_cast(gram_extractor_detail::mix64(key) & 0xFFFF); +} + +bool GramExtractor::is_boundary(uint8_t a, uint8_t b) const { + return boundary_hash16(a, b) < _boundary_threshold; +} + +void GramExtractor::_ascii_segment(std::string_view seg, std::vector* out) { + const size_t L = seg.size(); + const size_t n = _scheme.min_len; + if (L < n) { + return; + } + if (_scheme.mode == GramMode::DENSE) { + for (size_t i = 0; i + n <= L; i++) { + // Ruling R9: a candidate gram containing a NUL byte (0x00) is skipped entirely -- + // it is not emitted and does not affect the other windows, since computing the + // window boundaries does not depend on NUL at all. + std::string_view g = seg.substr(i, n); + if (g.find('\0') == std::string_view::npos) { + out->push_back(g); + } + } + return; + } + // SPARSE: content-defined chunking (CDC). Starting at boundary k, extend to the first later + // boundary j with j+2-k >= min_len, giving gram = [k, j+2); if no such boundary exists + // within max_len, take the full window [k, k+max_len); if even the full window does not fit + // (fewer than max_len bytes left before the end), this start position emits nothing. + const size_t maxlen = _scheme.max_len; + _is_boundary_at.assign(L, 0); + for (size_t i = 0; i + 1 < L; i++) { + _is_boundary_at[i] = is_boundary((uint8_t)seg[i], (uint8_t)seg[i + 1]); + } + for (size_t k = 0; k + 1 < L; k++) { + if (!_is_boundary_at[k]) { + continue; + } + size_t end = 0; + for (size_t j = k + 1; j + 1 < L && j + 2 - k <= maxlen; j++) { + if (_is_boundary_at[j] && j + 2 - k >= n) { + end = j + 2; + break; + } + } + if (end == 0) { + if (k + maxlen <= L) { + end = k + maxlen; + } else { + continue; + } + } + // Ruling R9: same as above -- a candidate gram containing NUL is skipped, and the + // boundary (k/end) computation is unaffected. + std::string_view g = seg.substr(k, end - k); + if (g.find('\0') == std::string_view::npos) { + out->push_back(g); + } + } +} + +// Deduplicate in place, keeping exactly the first occurrence of every distinct gram and leaving +// the survivors in their original relative order. That output order is a contract, not an +// implementation detail: TOKENIZE() hands it to the user and the ordered assertions in +// gram_extractor_test.cpp pin it, so any reordering here would be user-visible. +// +// The slot table replaces an std::unordered_set that used to be built from +// scratch for every row. Both are one forward pass keeping first appearances -- the semantics are +// literally the same -- but the set paid ~40-48 bytes per candidate gram (a 32-byte libstdc++ +// node: next pointer, cached hash, string_view; plus its share of the bucket array) and allocated +// all of it per row. This table is a reused member of 4 bytes per slot, so in the steady state it +// allocates nothing at all. Measured against the set it is never slower on any row length, mode +// or corpus, and roughly 1.7x faster at the median. +// +// Layout: slots hold indices into `out`, kEmpty marks a free slot, and the capacity is a power of +// two of at least 2n, so the table is at most half full -- linear probing always finds a free +// slot and the loop below always terminates. +void GramExtractor::_dedupe(std::vector* out) { + constexpr uint32_t kEmpty = std::numeric_limits::max(); + const size_t n = out->size(); + if (n < 2) { + return; + } + // A candidate gram is at least one byte long, so n never exceeds the row length, and the row + // length is an int32_t on the tokenizer path (GramTokenizer::reset). Indices therefore always + // fit in a uint32_t, and kEmpty can never collide with a real one. + DCHECK_LT(n, static_cast(kEmpty)); + size_t capacity = 8; + while (capacity < n * 2) { + capacity <<= 1; + } + // assign() on a buffer that is already large enough fills in place; only a row longer than + // every row before it reallocates. + _dedupe_slots.assign(capacity, kEmpty); + const size_t mask = capacity - 1; + const std::hash hasher; + size_t kept = 0; + for (size_t r = 0; r < n; r++) { + const std::string_view gram = (*out)[r]; + size_t slot = hasher(gram) & mask; + while (true) { + const uint32_t occupant = _dedupe_slots[slot]; + if (occupant == kEmpty) { + // A slot stores the COMPACTED index, not the read cursor. Since kept <= r, the + // survivor has already been written to its final position by the time anyone can + // look it up, so (*out)[occupant] below is always a valid, settled gram -- which + // is what lets the whole dedupe run in one in-place pass. + _dedupe_slots[slot] = static_cast(kept); + (*out)[kept++] = gram; + break; + } + if ((*out)[occupant] == gram) { + break; // already seen: drop this occurrence, keep the earlier one + } + slot = (slot + 1) & mask; + } + } + out->resize(kept); +} + +void GramExtractor::extract(std::string_view value, std::vector* out) { + out->clear(); + if (_scheme.lower_case) { + _folded.assign(value.data(), value.size()); + for (auto& ch : _folded) { + if (ch >= 'A' && ch <= 'Z') { + ch = (char)(ch - 'A' + 'a'); + } + } + value = _folded; + } + size_t i = 0; + const size_t L = value.size(); + while (i < L) { + if ((unsigned char)value[i] < 0x80) { + // ASCII segment: scan forward to the first non-ASCII byte (or the end of the + // string) and split the whole segment into grams at once. + size_t j = i; + while (j < L && (unsigned char)value[j] < 0x80) { + j++; + } + _ascii_segment(value.substr(i, j - i), out); + i = j; + } else { + // Non-ASCII: skipped, and it separates the ASCII segments around it. + // + // This used to emit one term per code point, and that shape has no working size. + // A one-character term is too common to filter with -- measured on weibo, a + // four-character literal with 18 matching rows proposed 500,000 candidates once + // those postings were dropped to make the index fit. Every scheme that gives such + // a column a rarer term instead pays in vocabulary: pairs, content-defined chunks + // of code points, and pairs emitted only where both characters are common all + // reach a dictionary of 5.5-22.8 MB against an 11.9 MB budget for the whole + // index, because multi-character units in these scripts are combinatorially + // diverse in a way that ASCII grams capped at max_gram bytes are not. + // + // Declining the work keeps the index honest. The extractor serves both the write + // and the query side, so a literal's non-ASCII part contributes no gram to the + // query either: the candidate set only ever widens, the predicate re-checks every + // candidate row, and a wholly non-ASCII pattern compiles to no grams and runs as + // a scan. Segments written when these terms were indexed stay readable for the + // same reason -- a query simply stops asking for them. + i += gram_extractor_detail::codepoint_len(value.data() + i, L - i); + } + } + _dedupe(out); +} + +void GramExtractor::grams_of_literal(std::string_view s, std::vector* out) { + std::vector tmp; + extract(s, &tmp); + out->assign(tmp.begin(), tmp.end()); +} + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/gram_extractor.h b/be/src/storage/index/inverted/gram/gram_extractor.h new file mode 100644 index 00000000000000..c3ab9789fa790d --- /dev/null +++ b/be/src/storage/index/inverted/gram/gram_extractor.h @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include +#include +#include +#include + +#include "storage/index/inverted/gram/gram_scheme.h" + +namespace doris::segment_v2::gram { + +// Split one row of text into grams according to a GramScheme: DENSE uses a fixed-length sliding +// window, SPARSE uses a content-defined chunking (CDC) rule that cuts variable-length grams at +// byte-pair boundaries. The split rule depends only on local byte content (a window of at most +// max_len bytes starting at the current position), so when the query side re-extracts grams from +// a literal, the resulting set is necessarily a subset of what the index side extracted from the +// whole row -- which is what lets the compiler fold regex literals. +// +// Ruling R9: an emitted gram never contains a NUL byte (0x00). A candidate window that spans a +// NUL is skipped entirely and emits nothing, while the computation of the window boundaries +// themselves is unaffected (locality is not broken by it); a non-ASCII code point takes the +// 1-gram path and can never contain NUL, so it needs no extra handling. RegexGramCompiler +// likewise degrades a literal or class item containing NUL to a non-indexable unknown character +// (treated as anyChar), so index and query side stay consistent and a NUL byte is never treated +// as indexable by only one of them, which would filter matches away. +// The 16-bit content hash of one byte pair. A pair is a chunk boundary when this is below the +// scheme's density threshold, so it is also the density at which this position starts being a +// boundary -- which is what lets a density be solved from a column's own bytes rather than +// configured for it (see gram_density.h). +uint16_t boundary_hash16(uint8_t a, uint8_t b); + +class GramExtractor { +public: + explicit GramExtractor(const GramScheme& scheme); + + // Extract the grams of one column value; the returned string_view points either into the + // extractor's internal buffer (when lower_case) or into value itself, and stays valid until + // the next extract call. Duplicates within the row are removed, keeping a stable order of + // first appearance. With lower_case=true the input is ASCII-folded before splitting (the + // boundary hash is computed over the folded bytes). + void extract(std::string_view value, std::vector* out); + + // Used by the query side: returns only the grams whose window falls entirely inside s, by + // exactly the same rule as extract -- a window emitted by extract always falls entirely + // inside the ASCII segment (or single code point) it belongs to, so this is equivalent to + // extract, except that it returns std::string values with an independent lifetime, for the + // compiler to fold the literal fragments of a regex. + void grams_of_literal(std::string_view s, std::vector* out); + + const GramScheme& scheme() const { return _scheme; } + + // Retunes the boundary rate. The write path calls this once, after solving the density + // from the column's own bytes and before any of that segment's rows are tokenized, so + // every row of a segment is cut at one rate and the segment records the rate it used. + // Nothing may call it once rows have been emitted: the query side reconstructs a segment's + // grams from that recorded rate, and two rates inside one segment would leave the rows cut + // at the other one unreachable. + void set_density_permille(uint16_t density_permille); + + // Bytes of scratch capacity this extractor keeps between calls. All three buffers are sized + // by the longest row seen so far and are never shrunk, so this is the extractor's steady + // resident cost -- which is what the SNII writer mirrors into its MemoryReporter. It is not + // the peak of one extract() call: `out` belongs to the caller and is counted there. + size_t reserved_bytes() const { + return _folded.capacity() + _is_boundary_at.capacity() + + _dedupe_slots.capacity() * sizeof(uint32_t); + } + + // Boundary test for one byte pair. Compute the deterministic hash on demand: tokenizers + // are created per indexed value, so enumerating all 65536 pairs in the constructor would + // dominate short-row extraction (and do entirely unused work in DENSE mode). + bool is_boundary(uint8_t a, uint8_t b) const; + +private: + // Split one pure-ASCII segment into grams per the scheme (DENSE fixed-length sliding window + // / SPARSE CDC rule). + void _ascii_segment(std::string_view seg, std::vector* out); + // Deduplicate within the row, preserving the order of first appearance. + void _dedupe(std::vector* out); + + GramScheme _scheme; + uint64_t _boundary_threshold; + std::string _folded; // ASCII-folded copy used when lower_case; output views may point here + std::vector _is_boundary_at; // per-position boundary flags reused in SPARSE mode + // Open-addressed slot table used by _dedupe, reused across rows. Holds indices into the + // caller's output vector, not the grams themselves; see _dedupe for the layout. + std::vector _dedupe_slots; +}; + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/gram_family.cpp b/be/src/storage/index/inverted/gram/gram_family.cpp new file mode 100644 index 00000000000000..f4fadb14ff3c96 --- /dev/null +++ b/be/src/storage/index/inverted/gram/gram_family.cpp @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/gram/gram_family.h" + +#include "runtime/index_policy/index_policy_mgr.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/analyzer/analyzer_provider.h" +#include "storage/index/inverted/inverted_index_parser.h" + +// This file (the sole exception under gram/) depends on the concrete runtime/index_policy and +// analyzer types, because the chain "resolve the analyzer name -> obtain the provider -> read its +// gram scheme" is runtime policy resolution and cannot be done without those dependencies +// (R8: every other file under gram/ stays free of any runtime dependency). + +namespace doris::segment_v2::gram { + +std::optional resolve_gram_scheme( + const std::map& index_properties, IndexPolicyMgr* mgr) { + const std::string name = get_analyzer_name_from_properties(index_properties); + // Built-in analyzer names (standard/english/unicode/chinese/icu/...) must short-circuit + // before the policy manager is touched: they are never registered as policies, and + // get_analyzer_provider_by_name would throw "Policy not found" straight away. The test + // reuses InvertedIndexAnalyzer::is_builtin_analyzer -- the same predicate + // create_analyzer_provider uses, so the two sides cannot drift on "what counts as a built-in + // name". + if (name.empty() || inverted_index::InvertedIndexAnalyzer::is_builtin_analyzer(name) || + mgr == nullptr) { + return std::nullopt; + } + // get_analyzer_provider_by_name throws when the policy is missing (it never returns + // nullptr); see the contract in the header. + return mgr->get_analyzer_provider_by_name(name)->gram_scheme(); +} + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/gram_family.h b/be/src/storage/index/inverted/gram/gram_family.h new file mode 100644 index 00000000000000..a1914904b81019 --- /dev/null +++ b/be/src/storage/index/inverted/gram/gram_family.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "storage/index/inverted/gram/gram_scheme.h" + +namespace doris { +class IndexPolicyMgr; +} + +namespace doris::segment_v2::gram { + +// Resolve index properties -> analyzer/normalizer name -> policy -> gram scheme, so that a +// caller that only has the index properties and cannot reach an analyzer provider (the query +// side) can decide whether an index belongs to the "gram family" (an ngram tokenizer carrying a +// mode property) and obtain its GramScheme parameters. +// +// nullopt is returned when: the analyzer name is empty, the name is a built-in analyzer +// (standard/english/... -- these names never reach the policy manager and are handled by +// InvertedIndexAnalyzer::create_analyzer_provider itself), the policy manager is not ready, or +// the policy exists but is not in the gram family. +// +// An exception is thrown when the name is neither a built-in analyzer nor found in the policy +// manager: IndexPolicyMgr::get_analyzer_provider_by_name then throws "Policy not found". This +// function does not swallow it -- that is a genuine configuration error, and the caller (the +// query side in phase C) decides whether to fail or to fall back to a full scan. +// +// Note: the write side (the SNII writer) does not come through here. It creates the analyzer +// provider itself and simply asks that provider for gram_scheme(), which saves one policy +// resolution and guarantees that "the analyzer actually used for tokenization" and "the analyzer +// judged to be gram family" are always the same object (Ruling R21). +std::optional resolve_gram_scheme( + const std::map& index_properties, IndexPolicyMgr* mgr); + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/gram_query.cpp b/be/src/storage/index/inverted/gram/gram_query.cpp new file mode 100644 index 00000000000000..55d349fd80ccc4 --- /dev/null +++ b/be/src/storage/index/inverted/gram/gram_query.cpp @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/gram/gram_query.h" + +#include +#include + +#include "util/url_coding.h" + +namespace doris::segment_v2::gram { + +// The BE storage target enables CMake unity builds (several .cpp files are compiled together, +// see UNITY_BUILD_BATCH_SIZE in be/src/storage/CMakeLists.txt), so every anonymous namespace in +// a batch is merged into one translation unit. A bare anonymous namespace then redefines any +// symbol whose name another file of the same batch happens to reuse (even in a different .cpp), +// and the batching changes as files are added to or removed from the directory, so "this batch +// only holds these files" cannot be assumed for long. Hence the extra named namespace private +// to this file, which isolates this file's anonymous namespace; the symbols inside it still +// have internal linkage (anonymous-namespace semantics are unaffected by a named enclosing +// namespace). +namespace gram_query_detail { + +namespace { + +// Deduplicate grams and sort them lexicographically: the canonical form of an AND/OR node's +// grams field. +void dedupe_grams(std::vector& v) { + std::sort(v.begin(), v.end()); + v.erase(std::unique(v.begin(), v.end()), v.end()); +} + +// Deduplicate sub-queries by their canonical structural key, keeping the first occurrence. +void dedupe_subs(std::vector& subs) { + std::set seen; + std::vector keep; + for (auto& s : subs) { + if (seen.insert(s.structural_key()).second) { + keep.push_back(std::move(s)); + } + } + subs = std::move(keep); +} + +// Whether the (already sorted) vector contains g. +bool has_gram(const std::vector& sorted, const std::string& g) { + return std::binary_search(sorted.begin(), sorted.end(), g); +} + +// Inside an OR node: if the gram set of one "pure gram set" child AND (one with no sub-queries) +// is a subset of another such child AND's, the latter is implied by the former (satisfying the +// stricter AND necessarily satisfies the looser one), so under OR semantics the stricter branch +// is redundant and should be dropped. Indices to drop are first marked in drop[] and only moved +// out once all comparisons are done, so that an object still needed by later comparisons is +// never moved away mid-loop. +void or_absorb_subsets(std::vector& subs) { + std::vector drop(subs.size(), 0); + for (size_t i = 0; i < subs.size(); i++) { + if (subs[i].op != GramQuery::Op::AND || !subs[i].subs.empty()) { + continue; + } + for (size_t j = 0; j < subs.size() && !drop[i]; j++) { + if (i == j || subs[j].op != GramQuery::Op::AND || !subs[j].subs.empty()) { + continue; + } + // A side with more grams cannot be a subset of the other, so skip it; when the + // counts are equal, compare only once (when j < i) so that two equal-sized identical + // sets are not both dropped for being subsets of each other (normally dedupe_subs + // has already removed such duplicates; this is only defensive). + if (subs[j].grams.size() > subs[i].grams.size() || + (subs[j].grams.size() == subs[i].grams.size() && j > i)) { + continue; + } + if (std::includes(subs[i].grams.begin(), subs[i].grams.end(), subs[j].grams.begin(), + subs[j].grams.end())) { + drop[i] = 1; + } + } + } + std::vector keep; + for (size_t i = 0; i < subs.size(); i++) { + if (!drop[i]) { + keep.push_back(std::move(subs[i])); + } + } + subs = std::move(keep); +} + +} // namespace + +} // namespace gram_query_detail + +GramQuery GramQuery::and_(GramQuery a, GramQuery b) { + if (a.is_none() || b.is_none()) { + return none(); + } + if (a.is_all()) { + return b; + } + if (b.is_all()) { + return a; + } + GramQuery r; + r.op = Op::AND; + // Flatten: when an operand is itself an AND, merge its grams/subs into r; otherwise keep it + // whole as a sub-query. + for (GramQuery* x : {&a, &b}) { + if (x->op == Op::AND) { + r.grams.insert(r.grams.end(), x->grams.begin(), x->grams.end()); + for (auto& s : x->subs) { + r.subs.push_back(std::move(s)); + } + } else { + r.subs.push_back(std::move(*x)); + } + } + gram_query_detail::dedupe_grams(r.grams); + // Absorption: if the AND already holds gram g, a child OR containing g is always true (an OR + // only needs one of its branches), so it no longer constrains the AND and can be dropped + // entirely. + std::vector keep; + for (auto& s : r.subs) { + bool absorbed = false; + if (s.op == Op::OR) { + for (const auto& g : s.grams) { + if (gram_query_detail::has_gram(r.grams, g)) { + absorbed = true; + break; + } + } + } + if (!absorbed) { + keep.push_back(std::move(s)); + } + } + r.subs = std::move(keep); + gram_query_detail::dedupe_subs(r.subs); + // Single-element collapse: an AND with no gram of its own and a single sub-query is + // equivalent to that sub-query. + if (r.grams.empty() && r.subs.size() == 1) { + return std::move(r.subs[0]); + } + return r; +} + +GramQuery GramQuery::or_(GramQuery a, GramQuery b) { + if (a.is_all() || b.is_all()) { + return all(); + } + if (a.is_none()) { + return b; + } + if (b.is_none()) { + return a; + } + GramQuery r; + r.op = Op::OR; + // Flatten: when an operand is an OR, merge its grams/subs; an AND holding a single gram (as + // produced by of_gram) is demoted to a gram leaf of this node; anything else is kept whole + // as a sub-query. + for (GramQuery* x : {&a, &b}) { + if (x->op == Op::OR) { + r.grams.insert(r.grams.end(), x->grams.begin(), x->grams.end()); + for (auto& s : x->subs) { + r.subs.push_back(std::move(s)); + } + } else if (x->op == Op::AND && x->grams.size() == 1 && x->subs.empty()) { + r.grams.push_back(x->grams[0]); + } else { + r.subs.push_back(std::move(*x)); + } + } + gram_query_detail::dedupe_grams(r.grams); + // Absorption: if the OR already holds gram g, a child AND containing g is implied by g (an + // AND requires all of its conditions, and one of them is already satisfied by another branch + // of the OR), so it no longer constrains the OR and can be dropped. + std::vector keep; + for (auto& s : r.subs) { + bool absorbed = false; + if (s.op == Op::AND) { + for (const auto& g : s.grams) { + if (gram_query_detail::has_gram(r.grams, g)) { + absorbed = true; + break; + } + } + } + if (!absorbed) { + keep.push_back(std::move(s)); + } + } + r.subs = std::move(keep); + gram_query_detail::dedupe_subs(r.subs); + gram_query_detail::or_absorb_subsets(r.subs); + // Single-element collapse. + if (r.grams.size() == 1 && r.subs.empty()) { + return of_gram(r.grams[0]); + } + if (r.grams.empty() && r.subs.size() == 1) { + return std::move(r.subs[0]); + } + return r; +} + +size_t GramQuery::leaf_count() const { + size_t c = grams.size(); + for (const auto& s : subs) { + c += s.leaf_count(); + } + return c; +} + +std::string GramQuery::structural_key() const { + if (op == Op::ALL) { + return "*"; + } + if (op == Op::NONE) { + return "!"; + } + std::string s = op == Op::AND ? "&(" : "|("; + bool first = true; + for (const auto& g : grams) { + if (!first) { + s += ','; + } + first = false; + std::string enc; + doris::base64_encode(g, &enc); + s += enc; + } + // Sub-queries are emitted sorted by their own structural_key() text, so identical structures + // always produce identical text. + std::vector ks; + for (const auto& c : subs) { + ks.push_back(c.structural_key()); + } + std::sort(ks.begin(), ks.end()); + for (const auto& k : ks) { + if (!first) { + s += ','; + } + first = false; + s += k; + } + return s + ")"; +} + +std::string GramQuery::to_debug_string() const { + if (op == Op::ALL) { + return "ALL"; + } + if (op == Op::NONE) { + return "NONE"; + } + std::string sep = op == Op::AND ? " & " : " | "; + std::string s = "("; + bool first = true; + for (const auto& g : grams) { + if (!first) { + s += sep; + } + first = false; + s += "\"" + g + "\""; + } + for (const auto& c : subs) { + if (!first) { + s += sep; + } + first = false; + s += c.to_debug_string(); + } + return s + ")"; +} + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/gram_query.h b/be/src/storage/index/inverted/gram/gram_query.h new file mode 100644 index 00000000000000..f92c3af5ffb901 --- /dev/null +++ b/be/src/storage/index/inverted/gram/gram_query.h @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include +#include +#include +#include + +namespace doris::segment_v2::gram { + +// Boolean query tree over grams: ALL/NONE are short-circuit constants, while an AND/OR node may +// carry gram leaves directly (the grams field) as well as sub-queries (the subs field) -- both +// are operands of that node's operator. and_/or_ are the only entry points for building a +// non-trivial query, and they simplify while constructing (flattening, deduplication, +// absorption, NONE/ALL short-circuits, single-element collapse), so a GramQuery instance is in +// simplified form at all times. A canonical structural key supports deduplication, while a +// readable debug string supports EXPLAIN. +struct GramQuery { + enum class Op : uint8_t { ALL, NONE, AND, OR }; + Op op = Op::ALL; + std::vector grams; // gram leaves held by this node (AND/OR only) + std::vector subs; // sub-queries (AND/OR only) + + static GramQuery all() { return GramQuery {}; } + static GramQuery none() { + GramQuery q; + q.op = Op::NONE; + return q; + } + // A single gram is modelled as an AND node holding one gram, so it goes through the same + // and_/or_ simplification logic. + static GramQuery of_gram(std::string g) { + GramQuery q; + q.op = Op::AND; + q.grams.push_back(std::move(g)); + return q; + } + // Build and simplify: flatten nested operators of the same kind, sort and deduplicate grams, + // apply absorption (a child OR containing a gram already present in the AND is absorbed, and + // a child AND containing a gram already present in the OR is absorbed), let a subset child + // AND absorb its supersets inside an OR, deduplicate structurally, short-circuit NONE/ALL, + // and collapse single-element nodes. + static GramQuery and_(GramQuery a, GramQuery b); + static GramQuery or_(GramQuery a, GramQuery b); + bool is_all() const { return op == Op::ALL; } + bool is_none() const { return op == Op::NONE; } + // Total number of gram leaves, counted recursively over all sub-queries. + size_t leaf_count() const; + // Readable form for EXPLAIN, e.g. "(\"abc\" & (\"de\" | \"fg\"))". + std::string to_debug_string() const; + // Canonical string key used for structural deduplication: two queries with the same + // structure (sub-query order does not matter, they are compared sorted) share the same key. + std::string structural_key() const; +}; + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/gram_scheme.cpp b/be/src/storage/index/inverted/gram/gram_scheme.cpp new file mode 100644 index 00000000000000..182aec064ceb45 --- /dev/null +++ b/be/src/storage/index/inverted/gram/gram_scheme.cpp @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/gram/gram_scheme.h" + +#include + +#include +#include + +namespace doris::segment_v2::gram { + +namespace { +// Parse an integer property whose value range is [lo, hi]; on failure return InvalidArgument +// with context. +Status parse_uint(const std::string& key, const std::string& v, uint32_t lo, uint32_t hi, + uint32_t* out) { + char* end = nullptr; + long x = std::strtol(v.c_str(), &end, 10); + if (end == v.c_str() || *end != '\0' || x < (long)lo || x > (long)hi) { + return Status::InvalidArgument("gram property {}={} out of range [{},{}]", key, v, lo, hi); + } + *out = (uint32_t)x; + return Status::OK(); +} + +// Parse a permille property: the input is a decimal in [lo, hi] (0.33, say) and is stored as +// that value times 1000 (330). +Status parse_permille(const std::string& key, const std::string& v, double lo, double hi, + uint32_t* out) { + char* end = nullptr; + double x = std::strtod(v.c_str(), &end); + if (end == v.c_str() || *end != '\0' || !(x >= lo && x <= hi)) { + return Status::InvalidArgument("gram property {}={} out of range [{},{}]", key, v, lo, hi); + } + *out = (uint32_t)std::lround(x * 1000.0); + return Status::OK(); +} +} // namespace + +Status GramScheme::from_properties(const std::map& props, + GramScheme* out) { + GramScheme s; + if (auto it = props.find("mode"); it != props.end()) { + if (it->second == "sparse" || it->second == "auto") { + s.mode = GramMode::SPARSE; // auto resolves per segment sample; sparse is the default + } else if (it->second == "dense") { + s.mode = GramMode::DENSE; + } else { + return Status::InvalidArgument("gram property mode={} must be auto|sparse|dense", + it->second); + } + } + if (auto it = props.find("min_gram"); it != props.end()) { + RETURN_IF_ERROR(parse_uint("min_gram", it->second, 1, 64, &s.min_len)); + } + if (auto it = props.find("max_gram"); it != props.end()) { + RETURN_IF_ERROR(parse_uint("max_gram", it->second, 1, 256, &s.max_len)); + } + if (s.max_len < s.min_len) { + return Status::InvalidArgument("gram property max_gram({}) < min_gram({})", s.max_len, + s.min_len); + } + if (auto it = props.find("density"); it != props.end()) { + RETURN_IF_ERROR(parse_permille("density", it->second, 0.001, 1.0, &s.density_permille)); + } + if (auto it = props.find("lower_case"); it != props.end()) { + if (it->second == "true" || it->second == "1") { + s.lower_case = true; + } else if (it->second == "false" || it->second == "0") { + s.lower_case = false; + } else { + return Status::InvalidArgument("gram property lower_case={} must be true|false", + it->second); + } + } + if (auto it = props.find("hash_version"); it != props.end()) { + RETURN_IF_ERROR(parse_uint("hash_version", it->second, 1, 1, &s.hash_version)); + } + *out = s; + return Status::OK(); +} + +std::map GramScheme::to_properties() const { + return {{"mode", mode == GramMode::DENSE ? "dense" : "sparse"}, + {"min_gram", std::to_string(min_len)}, + {"max_gram", std::to_string(max_len)}, + {"density", fmt::format("{:.3f}", density_permille / 1000.0)}, + {"lower_case", lower_case ? "true" : "false"}, + {"hash_version", std::to_string(hash_version)}}; +} + +std::string GramScheme::cache_key() const { + return fmt::format("gram:v{}:{}:{}:{}:{}:lc{}", hash_version, + mode == GramMode::DENSE ? "dense" : "sparse", min_len, max_len, + density_permille, lower_case ? 1 : 0); +} + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/gram_scheme.h b/be/src/storage/index/inverted/gram/gram_scheme.h new file mode 100644 index 00000000000000..4797a2119decdd --- /dev/null +++ b/be/src/storage/index/inverted/gram/gram_scheme.h @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include +#include +#include + +#include "common/status.h" + +namespace doris::segment_v2::gram { + +// Encoding mode of a gram scheme: DENSE emits every gram from a byte-by-byte sliding window, +// SPARSE emits hash-sampled sparse grams. auto is a third, user-visible value, but it exists +// only on the write side: after the sample is analysed it is stored as one of the two, so auto +// never appears in GramScheme (the source of truth for on-disk data and cache keys). +enum class GramMode : uint8_t { DENSE = 1, SPARSE = 2 }; + +// All parameters of a gram scheme, the single source of truth for downstream components such as +// GramExtractor and RegexGramCompiler. A GramScheme instance fully determines "which grams a +// given piece of text is split into", so it must be constructible from tokenizer/index +// properties, serializable back to properties (for the segment metadata), and able to produce a +// cache key. +struct GramScheme { + GramMode mode = GramMode::SPARSE; + uint32_t min_len = 3; // n (bytes) + uint32_t max_len = 16; // L (bytes, SPARSE only) + uint32_t density_permille = 250; // p x 1000 (SPARSE only) + bool lower_case = false; + uint32_t hash_version = 1; + + // Build a GramScheme from tokenizer/index properties; defaults are as above; an illegal + // value yields InvalidArgument. + static Status from_properties(const std::map& props, GramScheme* out); + // Write back to a property table, for persisting the segment metadata and as the input to + // the cache key computation. + std::map to_properties() const; + // Cache key of the form "gram:v1:sparse:3:16:250:lc0", uniquely identifying one set of + // scheme parameters. + std::string cache_key() const; + bool operator==(const GramScheme& o) const { + return mode == o.mode && min_len == o.min_len && max_len == o.max_len && + density_permille == o.density_permille && lower_case == o.lower_case && + hash_version == o.hash_version; + } +}; + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/regex_ast.cpp b/be/src/storage/index/inverted/gram/regex_ast.cpp new file mode 100644 index 00000000000000..1ee85f0ae72018 --- /dev/null +++ b/be/src/storage/index/inverted/gram/regex_ast.cpp @@ -0,0 +1,800 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/gram/regex_ast.h" + +#include +#include +#include + +namespace doris::segment_v2::gram { + +// The BE storage target enables CMake unity builds (several .cpp files are compiled together, +// see UNITY_BUILD_BATCH_SIZE in be/src/storage/CMakeLists.txt), so every anonymous namespace in +// a batch is merged into one translation unit. A bare anonymous namespace then redefines any +// symbol whose name another file of the same batch happens to reuse (even in a different .cpp), +// and the batching changes as files are added to or removed from the directory, so "this batch +// only holds these files" cannot be assumed for long. Hence the extra named namespace private +// to this file, which isolates this file's anonymous namespace; the symbols inside it still +// have internal linkage (anonymous-namespace semantics are unaffected by a named enclosing +// namespace). +namespace regex_ast_detail { + +namespace { + +// Maximum recursion nesting depth of `(...)` groups: every extra group level adds one more +// recursion through the parse_alt/parse_cat/parse_atom call chain. A malformed (or maliciously +// crafted) regex can drive that chain very deep with a pile of nested parentheses and blow the +// stack; this repository has already seen a stack overflow from deep recursion (CIR-21633), so +// there is a hard cap here that errors out instead of recursing further. +constexpr int kMaxNestingDepth = 64; + +// The parser derives conservative literal constraints for the scalar regex engines. +// Unsupported syntax fails parsing so the caller can skip gram filtering. + +// Infer the byte length of a UTF-8 sequence from its lead byte; an illegal lead byte counts as a +// single byte. +int utf8_len(unsigned char c) { + if (c < 0x80) { + return 1; + } + if ((c >> 5) == 0x6) { + return 2; + } + if ((c >> 4) == 0xE) { + return 3; + } + if ((c >> 3) == 0x1E) { + return 4; + } + return 1; // illegal lead byte: treat it as a single byte +} + +// The largest legal Unicode code point. Anything above it can only be a fake code point minted +// by decode_one_cp for an ill-formed byte, and must never reach encode_cp: the four-byte sequence +// encode_cp would produce encodes a value above U+10FFFF, so it is a byte string no encoder can +// emit and no index can hold, and demanding it as a gram would filter every row away. +constexpr uint32_t kMaxCodePoint = 0x10FFFF; + +// Decode the code point starting at s[0]; s must not be empty. A well-formed UTF-8 sequence +// yields its code point and its byte length; any ill-formed byte (an illegal lead byte, a +// truncated sequence or a bad continuation byte) yields the fake code point 0x110000+byte (still +// < 2^21, so it cannot collide with a legal one) and consumes exactly one byte. +// +// *consumed is what keeps a caller's cursor in sync with the decoder. Advancing by the length +// guessed from the lead byte instead would swallow the bytes following an ill-formed sequence -- +// regex metacharacters among them -- and silently compile a different pattern than the engine +// sees. +uint32_t decode_one_cp(std::string_view s, size_t* consumed) { + const auto c = static_cast(s[0]); + const int l = utf8_len(c); + *consumed = 1; + if (l == 1) { + return c < 0x80 ? c : 0x110000U + c; + } + if (static_cast(l) > s.size()) { + return 0x110000U + c; + } + uint32_t v = 0; + if (l == 2) { + v = c & 0x1FU; + } else if (l == 3) { + v = c & 0x0FU; + } else { + v = c & 0x07U; + } + for (int k = 1; k < l; k++) { + const auto cc = static_cast(s[k]); + if ((cc & 0xC0) != 0x80) { + return 0x110000U + c; + } + v = (v << 6) | (cc & 0x3FU); + } + *consumed = static_cast(l); + return v; +} + +// Encode one code point as UTF-8 and append it to out. +void encode_cp(uint32_t cp, std::string* out) { + if (cp < 0x80) { + out->push_back((char)cp); + } else if (cp < 0x800) { + out->push_back((char)(0xC0 | (cp >> 6))); + out->push_back((char)(0x80 | (cp & 0x3F))); + } else if (cp < 0x10000) { + out->push_back((char)(0xE0 | (cp >> 12))); + out->push_back((char)(0x80 | ((cp >> 6) & 0x3F))); + out->push_back((char)(0x80 | (cp & 0x3F))); + } else { + out->push_back((char)(0xF0 | (cp >> 18))); + out->push_back((char)(0x80 | ((cp >> 12) & 0x3F))); + out->push_back((char)(0x80 | ((cp >> 6) & 0x3F))); + out->push_back((char)(0x80 | (cp & 0x3F))); + } +} + +using NP = std::unique_ptr; + +NP mk(RegexNode::Type t) { + auto p = std::make_unique(); + p->type = t; + return p; +} + +// ASCII K and S also match the Kelvin sign and long s under the scalar engines' Unicode +// case-insensitive matching. Keep the same expansion for literals and small character classes. +void append_ascii_case_variants(uint32_t cp, std::vector* items) { + items->emplace_back(1, static_cast(cp)); + const uint32_t lower = cp >= 'A' && cp <= 'Z' ? cp + ('a' - 'A') : cp; + if (lower < 'a' || lower > 'z') { + return; + } + items->emplace_back(1, static_cast(cp == lower ? cp - ('a' - 'A') : lower)); + if (lower == 'k') { + items->emplace_back("K"); + } else if (lower == 's') { + items->emplace_back("ſ"); + } +} + +// Recursive-descent parser for the supported regex subset. +struct Parser { + std::string_view p; + size_t i = 0; + bool icase = false; + bool ok = true; + std::string err; + int depth = 0; // current group nesting depth, see kMaxNestingDepth + + explicit Parser(std::string_view s) : p(s) {} + + bool eof() const { return i >= p.size(); } + char peek() const { return eof() ? 0 : p[i]; } + + uint32_t next_cp(std::string* utf8) { + if (eof()) { + // Defensive fallback: every normal call site checks that a character is still + // available before entering next_cp; this merely distrusts the caller and avoids an + // out-of-bounds p[i] read at i==size() on a string_view, which -- unlike the + // std::string the prototype used -- is not guaranteed to be NUL-terminated. + utf8->clear(); + return 0; + } + // Advance by however many bytes the decoder actually consumed, never by the length + // guessed from the lead byte: an ill-formed sequence consumes exactly one byte, and + // advancing further would swallow the bytes that follow it -- including a regex + // metacharacter that may sit there -- and compile a pattern the engine never saw. + size_t consumed = 0; + const uint32_t cp = decode_one_cp(p.substr(i), &consumed); + *utf8 = std::string(p.substr(i, consumed)); + i += consumed; + return cp; + } + + NP parse() { + NP r = parse_alt(); + if (!eof()) { + ok = false; + err = "trailing input at " + std::to_string(i); + } + return r; + } + + NP parse_alt() { + std::vector branches; + branches.push_back(parse_cat()); + while (peek() == '|') { + i++; + branches.push_back(parse_cat()); + } + if (branches.size() == 1) { + return std::move(branches[0]); + } + NP a = mk(RegexNode::Type::ALT); + a->kids = std::move(branches); + return a; + } + + NP parse_cat() { + NP c = mk(RegexNode::Type::CAT); + while (!eof() && peek() != '|' && peek() != ')') { + if (peek() == '\\' && i + 1 < p.size() && p[i + 1] == 'Q') { + append_quoted_literals(&c->kids); + if (!c->kids.empty()) { + // A quote adds individual literal atoms. If it is empty, a following + // quantifier still applies to the preceding atom, including a group or + // repeat. Keep this token boundary: '+\\Q\\E?' must not become lazy '+?'. + c->kids.back() = parse_quant(std::move(c->kids.back())); + } + continue; + } + NP atom = parse_atom(); + if (!ok) { + return c; + } + if (!atom) { + continue; // e.g. a flags-only empty atom such as (?i) + } + atom = parse_quant(std::move(atom)); + c->kids.push_back(std::move(atom)); + } + return c; + } + + void append_quoted_literals(std::vector* atoms) { + i += 2; // '\\Q' + while (!eof() && !(peek() == '\\' && i + 1 < p.size() && p[i + 1] == 'E')) { + std::string utf8; + atoms->push_back(make_lit(next_cp(&utf8))); + } + if (!eof()) { + i += 2; // '\\E' + } + } + + NP parse_quant(NP a) { + while (!eof()) { + char c = peek(); + if (c == '*') { + i++; + NP s = mk(RegexNode::Type::STAR); + s->kids.push_back(std::move(a)); + a = std::move(s); + } else if (c == '+') { + i++; + NP s = mk(RegexNode::Type::PLUS); + s->kids.push_back(std::move(a)); + a = std::move(s); + } else if (c == '?') { + i++; + NP s = mk(RegexNode::Type::QUEST); + s->kids.push_back(std::move(a)); + a = std::move(s); + } else if (c == '{') { + size_t save = i; + i++; + int mn = 0; + int mx = -1; + bool has = false; + while (!eof() && std::isdigit(static_cast(peek()))) { + mn = mn * 10 + (peek() - '0'); + i++; + has = true; + } + if (!has) { + i = save; + break; + } + if (peek() == ',') { + i++; + if (std::isdigit(static_cast(peek()))) { + mx = 0; + while (!eof() && std::isdigit(static_cast(peek()))) { + mx = mx * 10 + (peek() - '0'); + i++; + } + } + } else { + mx = mn; + } + if (peek() != '}') { + i = save; + break; + } + i++; + NP s = mk(RegexNode::Type::REPEAT); + s->rmin = mn; + s->rmax = mx; + s->kids.push_back(std::move(a)); + a = std::move(s); + } else { + break; + } + if (peek() == '?') { + i++; // a lazy quantifier does not change the match set + } + } + return a; + } + + // Hex value of a `\x` escape in class_escape: on entry "\x" has already been consumed (i + // points at the brace or at the first hex digit). Ruling R12: the `\x{...}` form requires at + // least one hex digit inside the braces and the braces must be closed; the bare `\xHH` form + // requires exactly two hex digits, and anything shorter (end of string, or a non-hex + // character) is an error, matching RE2's rejection of `\x4`. On success the value is written + // to *v and true is returned; on failure ok=false and err are set and false is returned (the + // caller then returns immediately). Split out of class_escape to reduce its + // complexity/length; the semantics are identical to the original inline code. + bool parse_hex_escape_value(uint32_t* v) { + *v = 0; + if (peek() == '{') { + i++; + int cnt = 0; + while (!eof() && peek() != '}') { + if (!std::isxdigit(static_cast(peek()))) { + ok = false; + err = "bad \\x escape"; + return false; + } + *v = *v * 16 + + (std::isdigit(static_cast(peek())) + ? peek() - '0' + : (std::tolower(static_cast(peek())) - 'a' + 10)); + i++; + cnt++; + } + if (eof() || cnt == 0) { + ok = false; + err = "bad \\x escape"; + return false; + } + i++; // consume '}' + return true; + } + for (int cnt = 0; cnt < 2; cnt++) { + if (eof() || !std::isxdigit(static_cast(peek()))) { + ok = false; + err = "bad \\x escape"; + return false; + } + *v = *v * 16 + + (std::isdigit(static_cast(peek())) + ? peek() - '0' + : (std::tolower(static_cast(peek())) - 'a' + 10)); + i++; + } + return true; + } + + // Decode only character escapes whose meaning is shared by the scalar engines. Other + // letter/digit escapes may denote assertions, classes or backreferences; treating them as + // literals could exclude matching rows. Three-digit octal avoids short numeric escapes' + // ambiguity with backreferences. + bool parse_character_escape(uint32_t* cp) { + const char c = p[i++]; + switch (c) { + case 'a': + *cp = '\a'; + return true; + case 'f': + *cp = '\f'; + return true; + case 'n': + *cp = '\n'; + return true; + case 'r': + *cp = '\r'; + return true; + case 't': + *cp = '\t'; + return true; + case 'x': + return parse_hex_escape_value(cp); + default: + if (c >= '0' && c <= '7' && i + 1 < p.size() && p[i] >= '0' && p[i] <= '7' && + p[i + 1] >= '0' && p[i + 1] <= '7') { + *cp = (c - '0') * 64 + (p[i] - '0') * 8 + (p[i + 1] - '0'); + i += 2; + return true; + } + if (c >= ' ' && c <= '~' && !std::isalnum(static_cast(c))) { + *cp = c; + return true; + } + ok = false; + err = "unsupported character escape"; + return false; + } + } + + // Add one code point or escape from inside a class to out (a large class sets big). + void class_escape(std::vector* out, bool* big) { + // The caller always consumes the '\\' that triggered the escape before calling this + // function; being at eof here means that '\\' was the last character of the whole + // pattern with nothing left to escape -- the same error as "trailing backslash" in the + // top-level parse_atom, and handled the same way: report it instead of falling through + // to the default branch, where next_cp would read p[i] once more at i==p.size(). + if (eof()) { + ok = false; + err = "trailing backslash"; + return; + } + char c = peek(); + i++; + switch (c) { + case 'd': + case 'w': + case 's': + case 'D': + case 'W': + case 'S': + case 'v': // Hyperscan: vertical whitespace class; RE2: vertical tab. + *big = true; + break; + case 'p': + case 'P': + *big = true; + if (peek() == '{') { + while (!eof() && peek() != '}') { + i++; + } + i++; + } else { + i++; + } + break; + default: { + i--; + uint32_t cp; + if (!parse_character_escape(&cp)) { + return; + } + out->push_back(cp); + break; + } + } + } + + // Handling of a `-hi` range after a single code point lo inside [...]: a real range expands + // to [lo,hi] (beyond 4 items it degrades to a large class together with the items already + // collected); when it is not a range (the next character is not '-', or the '-' sits right + // before ']' and is therefore a literal '-'), lo itself becomes a standalone item. Split out + // of parse_class to reduce its complexity/length; the semantics are identical to the + // original inline code. + void parse_class_range_or_single(uint32_t lo, std::vector* items, bool* big) { + if (!(peek() == '-' && i + 1 < p.size() && p[i + 1] != ']')) { + items->push_back(lo); + return; + } + i++; + uint32_t hi; + if (peek() == '\\') { + i++; + std::vector tmp; + bool b2 = false; + class_escape(&tmp, &b2); + hi = tmp.empty() ? lo : tmp.back(); + if (b2) { + *big = true; + } + } else { + std::string u; + hi = next_cp(&u); + } + if (hi < lo) { + std::swap(lo, hi); + } + if (hi - lo + 1 + items->size() > 4) { + *big = true; + } else { + for (uint32_t x = lo; x <= hi; x++) { + items->push_back(x); + } + } + } + + // Encode the code points collected inside `[...]` into n->cls. Returns false when the class + // cannot be enumerated and the caller has to degrade it to big_class (with cls cleared, per + // the invariant documented on RegexNode): an ill-formed byte of the pattern, or a non-ASCII + // item under `(?i)`. Split out of parse_class to keep that function under the size + // threshold; the semantics are identical to the original inline loop. + bool encode_class_items(const std::vector& items, RegexNode* n) const { + for (auto cp : items) { + if (cp > kMaxCodePoint) { + // Same reason as in make_lit: an ill-formed byte of the pattern decodes to a + // fake code point that encode_cp would turn into a byte sequence no index can + // ever hold, so it cannot be enumerated as a class element. + return false; + } + if (icase && cp >= 128) { + // The scalar engines fold Unicode, while the gram index only folds ASCII. + // An unenumerated alternative makes this class unknown, but leaves surrounding + // literal constraints available to the compiler. + return false; + } + if (icase) { + append_ascii_case_variants(cp, &n->cls); + continue; + } + std::string u; + encode_cp(cp, &u); + n->cls.push_back(u); + } + return true; + } + + NP parse_class() { + // '[' has already been consumed + NP n = mk(RegexNode::Type::CLASS); + bool neg = false; + bool big = false; + if (peek() == '^') { + neg = true; + i++; + } + std::vector items; + bool first = true; + while (!eof() && (peek() != ']' || first)) { + first = false; + if (peek() == '[' && i + 1 < p.size() && p[i + 1] == ':') { // POSIX class, [:alpha:] + size_t e = p.find(":]", i); + if (e == std::string_view::npos) { + ok = false; + err = "bad posix class"; + return n; + } + i = e + 2; + big = true; + continue; + } + uint32_t lo; + if (peek() == '\\') { + i++; + size_t before = items.size(); + class_escape(&items, &big); + if (items.size() == before) { + continue; + } + lo = items.back(); + items.pop_back(); + } else { + std::string u; + lo = next_cp(&u); + } + parse_class_range_or_single(lo, &items, &big); + } + if (peek() != ']') { + ok = false; + err = "unterminated class"; + return n; + } + i++; + if (neg || big || items.size() > 4) { + n->big_class = true; + return n; + } + if (!encode_class_items(items, n.get())) { + n->big_class = true; + n->cls.clear(); + return n; + } + std::sort(n->cls.begin(), n->cls.end()); + n->cls.erase(std::unique(n->cls.begin(), n->cls.end()), n->cls.end()); + if (n->cls.size() > 4) { + // Under `(?i)` the case expansion can grow <= 4 original code points to more than + // 4 items (e.g. (?i)[abc] expands to 6), which degrades to a large class; cls must + // be cleared whenever big_class=true (see the invariant in the RegexNode comment in + // the header), otherwise a caller cannot tell "is this enumerable?" from big_class + // alone. + n->big_class = true; + n->cls.clear(); + } + return n; + } + + NP make_lit(uint32_t cp) const { + if (cp > kMaxCodePoint) { + // A fake code point minted for an ill-formed byte of the pattern. encode_cp would + // turn it into a four-byte sequence above U+10FFFF, while the index side only ever + // stores raw slices of the row, so that gram cannot exist anywhere and the AND would + // prune every row of the query. Degrade to the same "unknown character" the icase + // branch below uses. + return mk(RegexNode::Type::ANY); + } + if (icase && cp >= 128) { + // Unknown Unicode case variants must not become mandatory literal grams. + return mk(RegexNode::Type::ANY); + } + if (icase && ((cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z'))) { + NP n = mk(RegexNode::Type::CLASS); + append_ascii_case_variants(cp, &n->cls); + std::sort(n->cls.begin(), n->cls.end()); + return n; + } + NP n = mk(RegexNode::Type::LIT); + encode_cp(cp, &n->lit); + return n; + } + + // Decoding of the `\` escape branch in parse_atom: on entry '\\' has been consumed and eof + // has been ruled out (the caller handles a trailing backslash before switching here), and + // the escape is dispatched on the peek()ed character. Quotes are handled by parse_cat. + NP parse_backslash_escape() { + char e = peek(); + switch (e) { + case 'd': + case 'w': + case 's': + case 'D': + case 'W': + case 'S': + case 'v': { // Cover both Hyperscan's whitespace class and RE2's vertical tab. + i++; + NP n = mk(RegexNode::Type::CLASS); + n->big_class = true; + return n; + } + case 'b': + case 'B': + case 'A': + case 'Z': + case 'z': + i++; + return mk(RegexNode::Type::EMPTY); + case 'p': + case 'P': { + i++; + if (peek() == '{') { + while (!eof() && peek() != '}') { + i++; + } + i++; + } else { + i++; + } + NP n = mk(RegexNode::Type::CLASS); + n->big_class = true; + return n; + } + default: { + uint32_t cp; + if (!parse_character_escape(&cp)) { + return nullptr; + } + return make_lit(cp); + } + } + } + + // The '(' branch of parse_atom: parses a capturing group / a non-capturing group `(?:...)` / + // a named group `(?P...)` or `(?...)` and the inline flags `(?i) (?is) (?i:...)`, + // then recursively parses the group body and checks the closing parenthesis, all while + // maintaining the kMaxNestingDepth recursion cap. On entry the leading '(' has already been + // consumed. Groups restore the enclosing flags after parsing their bodies. + NP parse_group() { + const bool enclosing_icase = icase; + if (peek() == '?') { + i++; + if (peek() == ':') { + i++; + } else if (peek() == 'P' || peek() == '<') { // named group + if (peek() == 'P') { + i++; + } + if (peek() != '<') { + ok = false; + err = "bad group"; + return nullptr; + } + // `(?<=` and `(?', and a '>' inside the assertion body would + // leave the rest of that body to be parsed as an ordinary consuming + // subexpression. A lookbehind consumes nothing, so its body would become a + // mandatory literal and prune rows the engine really matches -- for example + // `aq>x(?<=q>x)y` would drop the row `aq>xy`. Give up on the whole pattern + // instead; the caller turns a parse failure into ALL, which filters nothing. + if (i + 1 < p.size() && (p[i + 1] == '=' || p[i + 1] == '!')) { + ok = false; + err = "lookbehind is not supported"; + return nullptr; + } + while (!eof() && peek() != '>') { + i++; + } + i++; + } else { // flags (?i) (?is) (?i:...) + bool neg = false; + while (!eof() && peek() != ')' && peek() != ':') { + char f = peek(); + i++; + if (f == '-') { + neg = true; + } else if (f == 'i') { + icase = !neg; + } else if (f == 's' || f == 'm' || f == 'U') { + // Recognized but irrelevant to the AST shape: (?s)(?m)(?U) do not + // change the match set of a literal or a class, so gram compilation + // need not distinguish them. + } else { + ok = false; + err = "unknown flag"; + return nullptr; + } + } + if (peek() == ')') { + i++; + // Flags-only groups affect the remainder of their enclosing scope. + return nullptr; + } + i++; // ':' + } + } + depth++; + if (depth > kMaxNestingDepth) { + ok = false; + err = "regex nesting too deep"; + depth--; + return nullptr; + } + NP inner = parse_alt(); + depth--; + icase = enclosing_icase; + if (peek() != ')') { + ok = false; + err = "missing )"; + return inner; + } + i++; + return inner; + } + + NP parse_atom() { + char c = peek(); + if (c == '(') { + i++; + return parse_group(); + } + if (c == '[') { + i++; + return parse_class(); + } + if (c == '.') { + i++; + return mk(RegexNode::Type::ANY); + } + if (c == '^' || c == '$') { + i++; + return mk(RegexNode::Type::EMPTY); + } + if (c == '\\') { + i++; + if (eof()) { + ok = false; + err = "trailing backslash"; + return nullptr; + } + return parse_backslash_escape(); + } + if (c == '*' || c == '+' || c == '?') { + ok = false; + err = "bad quantifier"; + return nullptr; + } + std::string u; + uint32_t cp = next_cp(&u); + return make_lit(cp); + } +}; + +} // namespace + +} // namespace regex_ast_detail + +Status parse_regex(std::string_view pattern, std::unique_ptr* root, + bool* case_insensitive) { + regex_ast_detail::Parser p(pattern); + std::unique_ptr r = p.parse(); + if (!p.ok) { + return Status::InvalidArgument("regex parse error: {}", p.err); + } + *case_insensitive = p.icase; + *root = std::move(r); + return Status::OK(); +} + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/regex_ast.h b/be/src/storage/index/inverted/gram/regex_ast.h new file mode 100644 index 00000000000000..f2a5696de0921d --- /dev/null +++ b/be/src/storage/index/inverted/gram/regex_ast.h @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris::segment_v2::gram { + +// A node of the regex syntax tree parsed out of the RE2 syntax subset. The tree keeps only what +// the gram compiler (Task 5: RegexGramCompiler) needs; it is not a full regex-engine AST: +// - LIT: the UTF-8 encoding of one code point; a multi-byte literal (CJK, say) stays one LIT +// for the whole code point and is never split down to bytes; +// - CLASS: a small class of <= 4 code points is expanded into cls (sorted, deduplicated); +// big_class=true marks a "non-enumerable" large class, negated class, +// `\d \w \s \D \W \S \pL \p{..}`, POSIX class and the like, and cls is then empty; +// - EMPTY: a node that adds no literal constraint (`^ $ \b \B \A \z`, a flags-only group, ...); +// - CAT/ALT/STAR/PLUS/QUEST: operator nodes whose semantics match the regex syntax, with +// children hanging off kids; +// - REPEAT: the bounded quantifier `{m}` / `{m,}` / `{m,n}`; rmax=-1 means no upper bound +// (`{m,}`). +// Under `(?i)`, ASCII letters expand into both cases, including the Unicode equivalents of +// K (Kelvin sign) and S (long s). Non-ASCII literals become ANY, and a small class containing +// a non-ASCII item becomes a big class: their Unicode case variants are not fully enumerated. +// This weakens only those leaves, preserving the constraints from surrounding literals. +struct RegexNode { + enum class Type : uint8_t { EMPTY, LIT, CLASS, ANY, CAT, ALT, STAR, PLUS, QUEST, REPEAT }; + Type type = Type::EMPTY; + std::string lit; // LIT: the UTF-8 of one code point + std::vector cls; // CLASS: <= 4 code points; empty + big_class = large/negated + bool big_class = false; + std::vector> kids; + int rmin = 0, rmax = -1; // REPEAT +}; + +// Parse a pattern in the RE2 syntax subset into a RegexNode AST. Supported: literals; escapes +// (`\. \a \f \n \t \r \ooo \xHH \x{...} \Q..\E`, where \ooo is three octal digits); +// classes (`[...]`, negation, ranges, POSIX classes, `\d \w \s \v \D \W \S \pL \p{..}`); +// `.`; groups (capturing, `(?:`, `(?P`, `(?`); +// the flags `(?i) (?s) (?m) (?U)` and `(?i:...)`; the quantifiers `* + ? {m} {m,} {m,n}` and +// their lazy suffixes; the anchors `^ $ \b \B \A \Z \z`; `|`. +// On success *root owns the whole tree and *case_insensitive says whether `(?i)` appeared in +// the pattern; on failure (syntax error, unclosed group/class, dangling escape, group nesting +// too deep, ...) Status::InvalidArgument is returned and neither *root nor *case_insensitive is +// guaranteed to have been written, so the caller must fall back conservatively (treat it as +// matching every row). +Status parse_regex(std::string_view pattern, std::unique_ptr* root, + bool* case_insensitive); + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/regex_gram_compiler.cpp b/be/src/storage/index/inverted/gram/regex_gram_compiler.cpp new file mode 100644 index 00000000000000..a4862b1183acf3 --- /dev/null +++ b/be/src/storage/index/inverted/gram/regex_gram_compiler.cpp @@ -0,0 +1,720 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/gram/regex_gram_compiler.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "common/logging.h" +#include "storage/index/inverted/gram/regex_ast.h" + +namespace doris::segment_v2::gram { + +// Storage is a unity build: other .cpp files in this directory also define file-level helpers +// named utf8_len / codepoint_len, so this file's helpers live in a file-specific namespace +// wrapped in an anonymous one. +namespace regex_gram_compiler_detail { +namespace { + +// An exact set whose strings are all already >= n yields useful grams more often, so it is +// demoted past 4 entries; together with kMaxExact (the cap that demotes regardless of length) +// these are the prototype simplify's two thresholds. +constexpr size_t kMaxLongExact = 4; +// Recursion depth cap for analyze. parse_regex only bounds group nesting (64 levels), while +// stacked quantifiers such as `a++++...` build an equally deep PLUS/REPEAT chain; past the cap +// we degrade to info_any_match (no constraint at all, conservative and safe) so a user-supplied +// pattern cannot blow the stack. +constexpr int kMaxAnalyzeDepth = 200; +// Maximum number of copies a REPEAT is unrolled into: x{m,..} is expanded exactly min(m, 4) +// times and then handled as a plus. +constexpr int kMaxRepeatUnroll = 4; + +// Byte length (1/2/3/4) of the UTF-8 sequence starting with lead byte c; an illegal lead byte +// counts as 1. +inline int utf8_len(unsigned char c) { + if (c < 0x80) { + return 1; + } + if ((c >> 5) == 0x6) { + return 2; + } + if ((c >> 4) == 0xE) { + return 3; + } + if ((c >> 3) == 0x1E) { + return 4; + } + return 1; +} + +// Byte length of one valid UTF-8 code point starting at p; returns 1 for a truncated sequence or +// a stray continuation byte. Exactly the rule GramExtractor uses: on the index side a non-ASCII +// code point is a whole 1-gram. +inline size_t codepoint_len(const char* p, size_t remain) { + int l = utf8_len((unsigned char)p[0]); + if (l == 1 || (size_t)l > remain) { + return 1; + } + for (int k = 1; k < l; k++) { + if (((unsigned char)p[k] & 0xC0) != 0x80) { + return 1; + } + } + return l; +} + +// The leading <= k bytes of s, always stopping on a code point boundary. Cutting a multi-byte +// code point in half would forge "fake grams" that do not exist in the index and cause false +// negatives, so we would rather keep a few bytes fewer. +std::string head_units(const std::string& s, size_t k) { + size_t i = 0; + while (i < s.size()) { + size_t l = codepoint_len(s.data() + i, s.size() - i); + if (i + l > k) { + break; + } + i += l; + } + return s.substr(0, i); +} + +// The trailing <= k bytes of s, always starting on a code point boundary (same reason as +// head_units). +std::string tail_units(const std::string& s, size_t k) { + if (s.size() <= k) { + return s; + } + size_t i = 0; + while (i < s.size() && s.size() - i > k) { + i += codepoint_len(s.data() + i, s.size() - i); + } + return s.substr(i); +} + +// Strict UTF-8 well-formedness test, the guard in front of "a literal becomes mandatory grams". +// +// The invariant it protects is the one the whole feature rests on: every gram the query side +// derives from a literal must be a gram the index side could have produced from a row that +// really matches. GramExtractor splits a row position-dependently -- an ASCII run is windowed, +// and every byte >= 0x80 is consumed as one whole code point -- so the same bytes are split +// differently depending on whether they start on a code point boundary. A literal that is not +// itself well-formed UTF-8 therefore yields grams that need not exist in a matching row, and +// requiring them prunes the row away. +// +// Rejected: a stray continuation byte (0x80-0xBF) as a lead byte, an illegal lead byte +// (0xF8-0xFF), a sequence that runs past the end of s, a bad continuation byte, an overlong +// encoding, a surrogate (U+D800-U+DFFF), and anything above U+10FFFF -- the last of which also +// catches whatever fake code point may still slip past the degradations in regex_ast.cpp. +bool is_well_formed_utf8(const std::string& s) { + size_t i = 0; + while (i < s.size()) { + const auto c = static_cast(s[i]); + if (c < 0x80) { + i++; + continue; + } + size_t len = 0; + uint32_t cp = 0; + uint32_t lowest = 0; // smallest code point this length may legally encode + if ((c & 0xE0) == 0xC0) { + len = 2; + cp = c & 0x1FU; + lowest = 0x80; + } else if ((c & 0xF0) == 0xE0) { + len = 3; + cp = c & 0x0FU; + lowest = 0x800; + } else if ((c & 0xF8) == 0xF0) { + len = 4; + cp = c & 0x07U; + lowest = 0x10000; + } else { + return false; // continuation byte used as a lead, or 0xF8-0xFF + } + if (i + len > s.size()) { + return false; // the sequence is not fully contained in s + } + for (size_t k = 1; k < len; k++) { + const auto cc = static_cast(s[i + k]); + if ((cc & 0xC0) != 0x80) { + return false; + } + cp = (cp << 6) | (cc & 0x3FU); + } + if (cp < lowest || cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) { + return false; + } + i += len; + } + return true; +} + +// Cartesian-product concatenation; sets *too_big and returns an empty set once the result would +// exceed kMaxSet (the caller then takes the demotion path). +std::set cross(const std::set& a, const std::set& b, + bool* too_big) { + std::set r; + *too_big = a.size() * b.size() > RegexGramCompiler::kMaxSet; + if (*too_big) { + return r; + } + for (const auto& x : a) { + for (const auto& y : b) { + r.insert(x + y); + } + } + return r; +} + +std::set uni(const std::set& a, const std::set& b) { + std::set r = a; + r.insert(b.begin(), b.end()); + return r; +} + +// Byte length of the shortest string in the set; an empty set counts as 0. +size_t min_str_len(const std::set& s) { + size_t m = SIZE_MAX; + for (const auto& x : s) { + m = std::min(m, x.size()); + } + return m == SIZE_MAX ? 0 : m; +} + +// Cox's five-tuple: a finite approximation of the string set a regex subtree can match. +// can_empty the subtree can match the empty string +// has_exact the match set is fully enumerated by exact (prefix/suffix are then meaningless) +// exact the fully enumerated set of matching strings +// prefix the possible beginnings of every matching string (nothing outside the set can +// start a match) +// suffix the possible endings of every matching string +// match the gram condition already known to hold inside every matching string +// Invariant: exact is non-empty when has_exact; otherwise prefix and suffix are both non-empty +// ("" meaning no constraint). +struct Info { + bool can_empty = false; + bool has_exact = false; + std::set exact; + std::set prefix; + std::set suffix; + GramQuery match; // defaults to ALL +}; + +// Matches the empty string only. +Info info_empty() { + Info i; + i.can_empty = true; + i.has_exact = true; + i.exact = {""}; + return i; +} + +// Matches exactly one unknown character (`.`, a big class, a non-indexable literal with NUL). +Info info_any_char() { + Info i; + i.prefix = {""}; + i.suffix = {""}; + return i; +} + +// Matches any string (`x*`, or a subtree we cannot reason about): imposes no constraint. +Info info_any_match() { + Info i; + i.can_empty = true; + i.prefix = {""}; + i.suffix = {""}; + return i; +} + +const std::set& pre(const Info& x) { + return x.has_exact ? x.exact : x.prefix; +} + +const std::set& suf(const Info& x) { + return x.has_exact ? x.exact : x.suffix; +} + +// The body of the Cox derivation. Every place that can produce grams goes through q_of_string / +// q_of_set, both of which return ALL when no gram is available, which guarantees "rather not +// filter at all than filter a match away". +class CoxAnalyzer { +public: + explicit CoxAnalyzer(GramExtractor& extractor) + : _extractor(extractor), _scheme(extractor.scheme()) {} + + // Whole tree -> gram query. + GramQuery compile(const RegexNode* root) { + Info info = _analyze(root, 0); + GramQuery m = std::move(info.match); + if (info.has_exact) { + m = GramQuery::and_(std::move(m), q_of_set(info.exact)); + } else { + m = GramQuery::and_(std::move(m), q_of_set(info.prefix)); + m = GramQuery::and_(std::move(m), q_of_set(info.suffix)); + } + return m; + } + + // AND of every gram of the literal s. Returns ALL when s yields no gram (too short, or no + // CDC boundary). + GramQuery q_of_string(const std::string& s) { + // The index side never produces a gram containing NUL, and neither may we (Ruling R9). + // analyze already treats a literal node holding NUL as anyChar; this is the fallback for + // other entry points such as compile_like. + // + // The UTF-8 test guards the same kind of mismatch for multi-byte content: this function + // is the single place where a literal turns into mandatory grams, so an ill-formed + // literal -- a LIKE segment that starts in the middle of a code point, raw illegal bytes + // in either kind of pattern -- has to degrade here rather than demand grams that a + // matching row may not hold. + if (s.find('\0') != std::string::npos || !is_well_formed_utf8(s)) { + return GramQuery::all(); + } + std::vector g; + _extractor.grams_of_literal(s, &g); + if (g.empty()) { + return GramQuery::all(); + } + GramQuery q; + q.op = GramQuery::Op::AND; + q.grams = std::move(g); + std::sort(q.grams.begin(), q.grams.end()); + q.grams.erase(std::unique(q.grams.begin(), q.grams.end()), q.grams.end()); + return q; + } + + // Any one string of the set suffices -> OR. An empty set means "nothing can match", hence + // NONE. + GramQuery q_of_set(const std::set& ss) { + if (ss.empty()) { + return GramQuery::none(); + } + GramQuery r = GramQuery::none(); + for (const auto& s : ss) { + r = GramQuery::or_(std::move(r), q_of_string(s)); + } + return r; + } + +private: + // How much of prefix/suffix to keep: a SPARSE gram is at most max_len long, so a whole gram + // has to fit; DENSE has fixed length n, so n-1 is enough (anything longer only repeats grams + // already folded into match). + size_t _keep() const { + if (_scheme.mode == GramMode::SPARSE) { + return _scheme.max_len; + } + return _scheme.min_len >= 1 ? _scheme.min_len - 1 : 0; + } + + // Whether every string in the set is long enough (>= n). Folding the whole set into match is + // only worth it when they all are: if a single string yields no gram, the OR collapses to + // ALL and the fold constrains nothing. + bool _all_long(const std::set& s) const { + return std::ranges::all_of(s, + [this](const auto& x) { return x.size() >= _scheme.min_len; }); + } + + // With scheme.lower_case the index side already folds ASCII letters to lower case, so + // literals must fold the same way. The whole pattern string must not be lowercased -- that + // would break the case-sensitive escapes `\E \B \W \D \S \P \A` -- so folding happens only + // on AST leaves (LIT / CLASS elements). + std::string _fold(const std::string& s) const { + if (!_scheme.lower_case) { + return s; + } + std::string r = s; + for (auto& ch : r) { + if (ch >= 'A' && ch <= 'Z') { + ch = static_cast(ch - 'A' + 'a'); + } + } + return r; + } + + // Fold the set's grams into match (only meaningful when every string is >= n, otherwise it + // carries no information), then trim the strings to keep bytes; if the set still exceeds + // kMaxSet after trimming, keep shrinking keep until it is small enough or trimmed to empty. + void _trim_set(std::set* s, GramQuery* match, bool is_suffix) { + if (s->empty()) { + return; + } + if (_all_long(*s)) { + *match = GramQuery::and_(std::move(*match), q_of_set(*s)); + } + size_t keep = _keep(); + for (;;) { + std::set t; + for (const auto& x : *s) { + t.insert(is_suffix ? tail_units(x, keep) : head_units(x, keep)); + } + *s = std::move(t); + if (s->size() <= RegexGramCompiler::kMaxSet || keep == 0) { + break; + } + keep--; + } + } + + // Demote exact to prefix/suffix: the full enumeration can no longer be maintained, but + // "every matching string starts with one of the exact strings and ends with one of them" + // still holds; fold the grams into match first so that information is not lost. + void _demote(Info* x) { + if (!x->has_exact) { + return; + } + if (_all_long(x->exact)) { + x->match = GramQuery::and_(std::move(x->match), q_of_set(x->exact)); + } + x->prefix = x->exact; + x->suffix = x->exact; + x->exact.clear(); + x->has_exact = false; + } + + // Demote exact once the set size / string length crosses a threshold, and trim + // prefix/suffix. The prototype's simplify also takes a `force` parameter, but every call + // site passes false, so it is omitted here. + Info _simplify(Info x) { + if (x.has_exact) { + const size_t ml = min_str_len(x.exact); + const bool all_long = _all_long(x.exact); + // Three demotion conditions: too many enumerated strings (kMaxExact); not too many, + // but each one already yields a gram, so enumerating further only multiplies OR + // branches (kMaxLongExact); the strings have grown to >= 2n, so more concatenation + // only makes exact longer without adding grams. In all three cases demoting to + // prefix/suffix and landing the grams we have in match pays off more. + if (x.exact.size() > RegexGramCompiler::kMaxExact || + (all_long && x.exact.size() > kMaxLongExact) || + ml >= static_cast(2) * _scheme.min_len) { + _demote(&x); + } + } + if (!x.has_exact) { + _trim_set(&x.prefix, &x.match, false); + _trim_set(&x.suffix, &x.match, true); + } + return x; + } + + // Concatenation xy. + Info _concat_info(Info x, Info y) { + Info r; + if (x.has_exact && y.has_exact) { + bool big = false; + std::set c = cross(x.exact, y.exact, &big); + if (!big) { + r.has_exact = true; + r.exact = std::move(c); + } else { + _demote(&x); + _demote(&y); + } + } + if (!r.has_exact) { + if (x.has_exact) { + bool big = false; + std::set c = cross(x.exact, y.prefix, &big); + if (big) { + _demote(&x); + r.prefix = x.prefix; + if (x.can_empty) { + r.prefix = uni(r.prefix, y.prefix); + } + } else { + r.prefix = std::move(c); + } + } else { + r.prefix = x.prefix; + if (x.can_empty) { + r.prefix = uni(r.prefix, pre(y)); + } + } + if (y.has_exact) { + bool big = false; + std::set c = cross(x.suffix, y.exact, &big); + if (big) { + _demote(&y); + r.suffix = y.suffix; + if (y.can_empty) { + r.suffix = uni(r.suffix, x.suffix); + } + } else { + r.suffix = std::move(c); + } + } else { + r.suffix = y.suffix; + if (y.can_empty) { + r.suffix = uni(r.suffix, suf(x)); + } + } + // Boundary grams: some suffix of x and some prefix of y are necessarily adjacent in + // a matching string, so their concatenation is necessarily a substring of it and can + // be folded straight into match. + if (!x.has_exact && !y.has_exact) { + bool big = false; + std::set c = cross(x.suffix, y.prefix, &big); + if (!big && !c.empty() && _all_long(c)) { + r.match = GramQuery::and_(std::move(r.match), q_of_set(c)); + } + } + } + r.match = GramQuery::and_(std::move(r.match), + GramQuery::and_(std::move(x.match), std::move(y.match))); + r.can_empty = x.can_empty && y.can_empty; + return _simplify(std::move(r)); + } + + // Alternation x|y. + Info _alt_info(Info x, Info y) { + Info r; + if (x.has_exact && y.has_exact) { + std::set u = uni(x.exact, y.exact); + if (u.size() <= RegexGramCompiler::kMaxSet) { + r.has_exact = true; + r.exact = std::move(u); + } else { + _demote(&x); + _demote(&y); + } + } + if (!r.has_exact) { + _demote(&x); + _demote(&y); + r.prefix = uni(x.prefix, y.prefix); + r.suffix = uni(x.suffix, y.suffix); + } + r.can_empty = x.can_empty || y.can_empty; + r.match = GramQuery::or_(std::move(x.match), std::move(y.match)); + return _simplify(std::move(r)); + } + + // x+: the repetition count is unknown, so all we can keep is "starts with one of x's + // strings and ends with one of x's strings". + Info _plus_info(Info x) { + _demote(&x); + return _simplify(std::move(x)); + } + + // Bounded quantifier REPEAT `{m}` / `{m,}` / `{m,n}`: unroll exactly + // min(m, kMaxRepeatUnroll) times (which captures the boundary grams across copies); when + // there may be more repetitions (rmax has no upper bound, or exceeds the unrolled count) + // fall back to plus and keep only the two ends. Split out of _analyze to reduce its + // complexity/length; the semantics are identical to the original switch case. + Info _repeat_info(const RegexNode* n, int depth) { + // rmin < 0 can only come from a counter overflow during parsing; the range is then + // untrustworthy, so be conservative. + if (n->kids.empty() || n->rmin < 0) { + return info_any_match(); + } + if (n->rmin == 0 && n->rmax == 0) { + return info_empty(); + } + if (n->rmin == 0) { + return info_any_match(); + } + Info acc = info_empty(); + const int reps = std::min(n->rmin, kMaxRepeatUnroll); + for (int k = 0; k < reps; k++) { + acc = _concat_info(std::move(acc), _analyze(n->kids[0].get(), depth + 1)); + } + if (n->rmax == n->rmin && n->rmin <= kMaxRepeatUnroll) { + return acc; + } + return _plus_info(std::move(acc)); + } + + Info _analyze(const RegexNode* n, int depth) { + if (n == nullptr || depth > kMaxAnalyzeDepth) { + return info_any_match(); + } + switch (n->type) { + case RegexNode::Type::EMPTY: + return info_empty(); + case RegexNode::Type::LIT: { + std::string lit = _fold(n->lit); + // Ruling R9: a literal containing NUL is not indexable; treat the whole node as one + // unknown character. + if (lit.find('\0') != std::string::npos) { + return info_any_char(); + } + Info i; + i.has_exact = true; + i.exact.insert(std::move(lit)); + return _simplify(std::move(i)); + } + case RegexNode::Type::CLASS: { + DCHECK(!n->big_class || n->cls.empty()); + if (n->big_class || n->cls.empty()) { + return info_any_char(); + } + Info i; + i.has_exact = true; + for (const auto& s : n->cls) { + std::string f = _fold(s); + // A single non-indexable element degrades the whole class to an unknown + // character: dropping just that element would stop exact from covering every + // possibility, and that is exactly where false negatives come from. + if (f.find('\0') != std::string::npos) { + return info_any_char(); + } + i.exact.insert(std::move(f)); // folding may create duplicates; the set dedups + } + return _simplify(std::move(i)); + } + case RegexNode::Type::ANY: + return info_any_char(); + case RegexNode::Type::CAT: { + Info acc = info_empty(); + for (const auto& k : n->kids) { + acc = _concat_info(std::move(acc), _analyze(k.get(), depth + 1)); + } + return acc; + } + case RegexNode::Type::ALT: { + if (n->kids.empty()) { + return info_any_match(); + } + Info acc = _analyze(n->kids[0].get(), depth + 1); + for (size_t k = 1; k < n->kids.size(); k++) { + acc = _alt_info(std::move(acc), _analyze(n->kids[k].get(), depth + 1)); + } + return acc; + } + case RegexNode::Type::STAR: + return info_any_match(); + case RegexNode::Type::PLUS: + if (n->kids.empty()) { + return info_any_match(); + } + return _plus_info(_analyze(n->kids[0].get(), depth + 1)); + case RegexNode::Type::QUEST: + if (n->kids.empty()) { + return info_any_match(); + } + return _alt_info(_analyze(n->kids[0].get(), depth + 1), info_empty()); + case RegexNode::Type::REPEAT: + return _repeat_info(n, depth); + } + return info_any_match(); + } + + GramExtractor& _extractor; + const GramScheme& _scheme; +}; + +} // namespace +} // namespace regex_gram_compiler_detail + +RegexGramCompiler::RegexGramCompiler(const GramScheme& scheme) : _extractor(scheme) {} + +void RegexGramCompiler::apply_gram_budget(GramQuery* q) { + if (q->leaf_count() > kMaxQueryGrams) { + *q = GramQuery::all(); + } +} + +Status RegexGramCompiler::compile_regexp(std::string_view pattern, GramQuery* out) { + // Hyperscan compiles a C string, while scalar string fast paths and RE2 use the full + // length. No constraints after a raw NUL are universally required across those paths. + if (pattern.find('\0') != std::string_view::npos) { + *out = GramQuery::all(); + return Status::OK(); + } + std::unique_ptr root; + bool case_insensitive = false; + // A parse failure always falls back conservatively to ALL; never return an error and fail + // the caller's query. + if (!parse_regex(pattern, &root, &case_insensitive).ok() || root == nullptr) { + *out = GramQuery::all(); + return Status::OK(); + } + // case_insensitive itself needs no extra handling: with lower_case=false parse_regex has + // already expanded ASCII letter literals under `(?i)` into CLASS{c,C} (Cox's approach); with + // lower_case=true both index and query fold to lower case, so it is enough for CoxAnalyzer + // to fold on the AST leaves. + regex_gram_compiler_detail::CoxAnalyzer analyzer(_extractor); + *out = analyzer.compile(root.get()); + apply_gram_budget(out); + return Status::OK(); +} + +Status RegexGramCompiler::compile_like(std::string_view like_pattern, GramQuery* out) { + // LIKE-to-regex conversion preserves raw NUL, so its Hyperscan path has the same + // C-string truncation as REGEXP. Length-aware scalar fast paths can still use all bytes. + if (like_pattern.find('\0') != std::string_view::npos) { + *out = GramQuery::all(); + return Status::OK(); + } + regex_gram_compiler_detail::CoxAnalyzer analyzer(_extractor); + GramQuery q = GramQuery::all(); + std::string seg; + // Wildcards cut the pattern into literal segments: the bytes inside one segment necessarily + // appear consecutively in a matching row, so grams can be taken from it directly. + auto flush = [&] { + if (!seg.empty()) { + q = GramQuery::and_(std::move(q), analyzer.q_of_string(seg)); + seg.clear(); + } + }; + for (size_t i = 0; i < like_pattern.size(); i++) { + const char c = like_pattern[i]; + if (c == '\\') { + if (i + 1 < like_pattern.size()) { + const char next = like_pattern[i + 1]; + if (next == '%' || next == '_' || next == '\\') { + // Doris LIKE really escapes only these three: the backslash is consumed and + // next joins the current segment as a literal (next and its neighbours + // necessarily appear consecutively in a matching row). + seg.push_back(next); + i++; + continue; + } + // `\x` (x not one of % _ \): it is unclear whether the engine keeps the + // backslash itself (two bytes "\x" in the row) or drops it (just "x" in the + // row, the old implementation's assumption). All that holds under both + // readings is "x is adjacent to whatever follows it"; whether a backslash sits + // between x and what precedes it is unknown, so cut the segment at the + // backslash -- the backslash yields no gram, and x starts a new segment on the + // next iteration instead of joining the segment already cut (this loses a + // little pruning power but can never filter a match away). + flush(); + continue; + } + // A lone trailing backslash with nothing to escape: cut the segment and ignore it. + flush(); + continue; + } + if (c == '%' || c == '_') { + flush(); + continue; + } + seg.push_back(c); + } + flush(); + *out = std::move(q); + apply_gram_budget(out); + return Status::OK(); +} + +} // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/regex_gram_compiler.h b/be/src/storage/index/inverted/gram/regex_gram_compiler.h new file mode 100644 index 00000000000000..df50ad2a43998b --- /dev/null +++ b/be/src/storage/index/inverted/gram/regex_gram_compiler.h @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/gram/gram_extractor.h" +#include "storage/index/inverted/gram/gram_query.h" +#include "storage/index/inverted/gram/gram_scheme.h" + +namespace doris::segment_v2::gram { + +// Compiles a regex / LIKE pattern into a boolean query over grams (Cox's "five-tuple" +// derivation: can_empty / exact / prefix / suffix / match). +// +// The one hard invariant is "false positives are fine, false negatives are not": for any row r, +// if the pattern matches r then the compiled GramQuery is necessarily true over "the gram set +// extracted from r with the same GramScheme". So anywhere the derivation cannot be made reliably +// (a parse failure, an oversized character class, a set explosion, a literal shorter than one +// gram, ...) it degrades to ALL (filtering no row at all) and never emits a stronger condition +// that could filter a matching row away. +// +// The compiler only folds literals into grams and makes no regex-semantic decisions: the real +// row-level matching is still done by the regexp / like expression above it, and this query only +// serves to skip rows that cannot possibly match. +// +// An instance holds GramExtractor's internal buffers, so both compile_* methods are non-const +// and not thread-safe: every thread that uses one constructs its own. Construction is O(1). +class RegexGramCompiler { +public: + explicit RegexGramCompiler(const GramScheme& scheme); + + // Compile a regex. Any parse or derivation failure returns OK with *out = ALL (a + // conservative fall back to a full scan); only an internal assertion failure returns non-OK. + Status compile_regexp(std::string_view pattern, GramQuery* out); + + // Compile a LIKE pattern: cut it into literal segments at % and _, AND the grams of each + // segment together and then AND the segments; an all-wildcard pattern (no literal segment) + // yields ALL. + // + // The escape character is fixed to `\`, and only `\%`, `\_` and `\\` count as escapes (which + // matches Doris LIKE's actual semantics); the escaped character joins the current segment as + // a literal. For any other `\x` (x not one of those three) it is undecidable whether the + // engine keeps the backslash itself, so under either reading the segment is conservatively + // cut at the backslash and no gram spans that point, while x still starts the next segment + // and takes part in the following literal (this only loses a little pruning power and can + // never filter a match away); a lone trailing `\` likewise cuts the segment and is ignored. + // The `ESCAPE` clause is not supported: the literal/wildcard split here always assumes `\` + // is the escape character, so a caller facing a custom escape character other than `\` must + // skip the index -- otherwise the derived boundaries would disagree with the engine's actual + // semantics and could filter genuinely matching rows away. + // + // ILIKE semantics are decided by the caller through scheme.lower_case (index and query fold + // together). + Status compile_like(std::string_view like_pattern, GramQuery* out); + + // Size cap of the exact/prefix/suffix sets: a Cartesian product or union larger than this is + // demoted, which keeps the enumeration from exploding. + static constexpr size_t kMaxSet = 20; + // An exact set larger than this is demoted to prefix/suffix (after folding the grams it + // already yields into match). + static constexpr size_t kMaxExact = 7; + // Total number of gram leaves the whole compiled tree may carry. kMaxSet bounds a single + // set, which does not bound the tree: a wide alternation can multiply several bounded sets + // into a query with more than a hundred leaves. Every leaf costs at least one term dictionary + // lookup and one posting read, and on object storage each of those is a separate remote + // request, so such a query can spend far more on index IO than the scan it is supposed to + // save. Over the budget the query degrades to ALL and the scan runs unaided -- giving up the + // acceleration, never the correctness. + static constexpr size_t kMaxQueryGrams = 64; + +private: + // Degrade *q to ALL when its gram leaves exceed kMaxQueryGrams. Applied to the result of both + // compile entry points, so no caller can be handed a query whose index IO outweighs the scan. + static void apply_gram_budget(GramQuery* q); + + // The extractor also owns the scheme (_extractor.scheme()), so the two configurations cannot + // disagree. + GramExtractor _extractor; +}; + +} // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/gram_density_corpus_test.cpp b/be/test/storage/index/inverted/gram/gram_density_corpus_test.cpp new file mode 100644 index 00000000000000..1d997578692c3a --- /dev/null +++ b/be/test/storage/index/inverted/gram/gram_density_corpus_test.cpp @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// What the solver actually answers on real columns, and what it costs. +// +// A rule that adapts is only worth the name if it lands somewhere different on data that is +// different, and if what it lands on is defensible. This prints, for each corpus and each +// promise, the solved density, the grams per byte it produces, and the share of literals of +// the promised length it really covers -- so the claim can be checked rather than believed. +// +// Corpus-driven and skipped without one: +// GRAM_DENSITY_CORPUS=/path/to/lines.txt GRAM_DENSITY_ROWS=200000 + +#include + +#include +#include +#include +#include +#include + +#include "storage/index/inverted/gram/gram_density.h" +#include "storage/index/inverted/gram/gram_extractor.h" +#include "storage/index/inverted/gram/gram_scheme.h" + +namespace doris::segment_v2::gram { +namespace { + +std::vector read_rows(const char* path, size_t rows) { + std::vector out; + std::ifstream in(path); + std::string line; + while (out.size() < rows && std::getline(in, line)) { + if (!line.empty()) { + out.push_back(line); + } + } + return out; +} + +GramScheme scheme_at(uint16_t density) { + GramScheme s; + s.mode = GramMode::SPARSE; + s.min_len = 3; + s.max_len = 4; + s.density_permille = density; + return s; +} + +struct Outcome { + double grams_per_byte = 0.0; + double covered = 0.0; +}; + +Outcome evaluate(const std::vector& rows, uint16_t density, size_t len) { + GramExtractor extractor(scheme_at(density)); + std::vector grams; + size_t total_grams = 0; + size_t total_bytes = 0; + size_t windows = 0; + size_t covered = 0; + for (const std::string& row : rows) { + extractor.extract(row, &grams); + total_grams += grams.size(); + total_bytes += row.size(); + // Coverage is measured the way a query experiences it: take the row's own substrings + // of the promised length and ask whether the extractor produces anything for them. + for (size_t start = 0; start + len <= row.size(); start += len) { + ++windows; + extractor.extract(row.substr(start, len), &grams); + covered += grams.empty() ? 0 : 1; + } + } + return {total_bytes == 0 ? 0.0 : static_cast(total_grams) / total_bytes, + windows == 0 ? 0.0 : static_cast(covered) / windows}; +} + +} // namespace + +TEST(GramDensityCorpusTest, ReportsWhatEachPromiseSolvesTo) { + const char* path = std::getenv("GRAM_DENSITY_CORPUS"); + if (path == nullptr) { + GTEST_SKIP() << "set GRAM_DENSITY_CORPUS to a line-per-record corpus"; + } + const char* rows_env = std::getenv("GRAM_DENSITY_ROWS"); + const size_t want = rows_env != nullptr ? std::strtoul(rows_env, nullptr, 10) : 200000; + const std::vector rows = read_rows(path, want); + ASSERT_FALSE(rows.empty()) << path; + + size_t bytes = 0; + for (const std::string& r : rows) { + bytes += r.size(); + } + printf("\ncorpus: %s, %zu rows, %.2f MB, %.1f B/row\n", path, rows.size(), bytes / 1e6, + static_cast(bytes) / rows.size()); + printf("%10s %8s %10s %12s %11s %s\n", "promise L", "q", "solved", "grams/byte", "covered", + "vs fixed 0.25"); + for (const size_t len : {8U, 12U, 16U, 24U}) { + for (const uint32_t q : {950U, 990U}) { + const uint16_t solved = solve_density_permille(rows, len, 4, q); + const Outcome got = evaluate(rows, solved, len); + const Outcome fixed = evaluate(rows, 250, len); + printf("%10zu %8.3f %10.3f %12.3f %10.1f%% grams/byte %.3f, covered %.1f%%\n", len, + q / 1000.0, solved / 1000.0, got.grams_per_byte, 100.0 * got.covered, + fixed.grams_per_byte, 100.0 * fixed.covered); + EXPECT_GE(solved, kMinSolvedDensityPermille); + EXPECT_LE(solved, kMaxSolvedDensityPermille); + } + } +} + +} // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/gram_density_test.cpp b/be/test/storage/index/inverted/gram/gram_density_test.cpp new file mode 100644 index 00000000000000..a5109b6a1aa900 --- /dev/null +++ b/be/test/storage/index/inverted/gram/gram_density_test.cpp @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Density is solved from the column, not chosen for it. +// +// The knob was a configured number, and every value picked for it was wrong somewhere: 0.25 +// costs more to write and store than the data needs, 0.10 silently stops indexing a 21-byte +// literal. Both failures come from the same mistake -- a boundary rate is being set without +// looking at the bytes it will be applied to, and the same rate lands differently on +// different data. Measured at a nominal 0.25, the realised grams per byte were 0.215 on log +// text, 0.288 on URL paths and 0.183 on agent traces: a 1.6x spread from one number. +// +// What the index actually promises is "a literal of at least L bytes can be found". Given +// that promise, the rate is not a choice. A window of L bytes is indexable exactly when it +// contains a boundary, a boundary is decided by the identity of a byte pair, so the smallest +// hash among a window's pairs decides the rate at which that window becomes indexable. Take +// those minima over the column's own text and the sparsest rate keeping the promise for a +// share q of windows is the q-quantile of them. One pass, exact, and it comes out different +// on different data because the data is different. + +#include "storage/index/inverted/gram/gram_density.h" + +#include + +#include +#include +#include + +#include "storage/index/inverted/gram/gram_extractor.h" +#include "storage/index/inverted/gram/gram_scheme.h" + +namespace doris::segment_v2::gram { +namespace { + +GramScheme scheme_at(uint16_t density_permille) { + GramScheme s; + s.mode = GramMode::SPARSE; + s.min_len = 3; + s.max_len = 4; + s.density_permille = density_permille; + return s; +} + +// The share of L-byte windows of `rows` that contain at least one boundary at this density -- +// i.e. the share of literals of that length the index would be able to serve. +double covered_share(const std::vector& rows, size_t len, uint16_t density) { + GramExtractor extractor(scheme_at(density)); + size_t total = 0; + size_t covered = 0; + for (const std::string& row : rows) { + if (row.size() < len) { + continue; + } + for (size_t start = 0; start + len <= row.size(); ++start) { + ++total; + bool has = false; + // Only positions with a whole gram's room left inside the window: a boundary + // nearer the end than max_len emits nothing, so counting it would credit + // coverage the extractor does not actually deliver. + for (size_t i = start; i + 4 <= start + len; ++i) { + if (extractor.is_boundary(static_cast(row[i]), + static_cast(row[i + 1]))) { + has = true; + break; + } + } + covered += has ? 1 : 0; + } + } + return total == 0 ? 0.0 : static_cast(covered) / static_cast(total); +} + +std::vector log_like_rows() { + std::vector rows; + for (int i = 0; i < 400; ++i) { + rows.push_back("rpc error: code = Unavailable desc = transport closing conn " + + std::to_string(i * 7919)); + } + return rows; +} + +// Low-entropy text: a handful of path components repeated. The same byte pairs recur, so a +// window's pairs are far from independent and a rate solved on log text is wrong here. +std::vector url_like_rows() { + static const char* parts[] = {"/images/", "/french/", "/english/", "/history/", "/docs/"}; + std::vector rows; + for (int i = 0; i < 400; ++i) { + rows.push_back(std::string("GET ") + parts[i % 5] + parts[(i / 5) % 5] + "index.html"); + } + return rows; +} + +} // namespace + +// The promise is kept: at the solved density, at least the requested share of windows of the +// promised length really do contain a boundary. +TEST(GramDensityTest, SolvedDensityKeepsThePromise) { + for (const auto& rows : {log_like_rows(), url_like_rows()}) { + for (const size_t len : {8U, 12U, 16U}) { + for (const uint32_t q : {900U, 950U, 990U}) { + const uint16_t density = solve_density_permille(rows, len, 4, q); + const double share = covered_share(rows, len, density); + EXPECT_GE(share, q / 1000.0 - 0.02) + << "len=" << len << " q=" << q << " solved density=" << density; + } + } + } +} + +// And it is the SPARSEST rate that keeps it: one notch lower stops keeping it. This is what +// makes the answer a solution rather than a safe over-estimate -- an index any larger than +// this is paying for coverage it was not asked for. +TEST(GramDensityTest, SolvedDensityIsNotLargerThanItNeedsToBe) { + const std::vector rows = log_like_rows(); + const uint32_t q = 950; + const size_t len = 12; + const uint16_t density = solve_density_permille(rows, len, 4, q); + ASSERT_GT(density, 1U) << "the fixture must not solve to the floor for this to mean anything"; + const double at = covered_share(rows, len, density); + const double below = covered_share(rows, len, static_cast(density / 2)); + EXPECT_GE(at, q / 1000.0 - 0.02); + EXPECT_LT(below, at) << "halving the solved density must cost coverage, or the solved " + "value was not the sparsest one that works"; +} + +// The whole point: the same promise resolves to different rates on different columns. +TEST(GramDensityTest, DifferentDataSolvesToDifferentDensity) { + const uint16_t on_logs = solve_density_permille(log_like_rows(), 12, 4, 950); + const uint16_t on_urls = solve_density_permille(url_like_rows(), 12, 4, 950); + EXPECT_NE(on_logs, on_urls) + << "a rate solved from the data cannot come out identical on data this different " + "(logs=" + << on_logs << ", urls=" << on_urls << ")"; +} + +// A longer promise is cheaper to keep: more pairs in the window, so a sparser rate suffices. +TEST(GramDensityTest, LongerPromisedLiteralsAllowASparserIndex) { + const std::vector rows = log_like_rows(); + const uint16_t at8 = solve_density_permille(rows, 8, 4, 950); + const uint16_t at24 = solve_density_permille(rows, 24, 4, 950); + EXPECT_LT(at24, at8) << "at8=" << at8 << " at24=" << at24; +} + +// Degenerate inputs must not produce a density that indexes nothing or everything. +TEST(GramDensityTest, DegenerateInputFallsBackWithinBounds) { + EXPECT_GE(solve_density_permille({}, 12, 4, 950), kMinSolvedDensityPermille); + EXPECT_LE(solve_density_permille({}, 12, 4, 950), kMaxSolvedDensityPermille); + // Rows shorter than the promised length carry no window of that length at all. + EXPECT_GE(solve_density_permille({"ab", "cd"}, 12, 4, 950), kMinSolvedDensityPermille); + // A demand for total coverage cannot exceed the ceiling. + EXPECT_LE(solve_density_permille(log_like_rows(), 12, 4, 1000), kMaxSolvedDensityPermille); +} + +} // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/gram_df_distribution_test.cpp b/be/test/storage/index/inverted/gram/gram_df_distribution_test.cpp new file mode 100644 index 00000000000000..d6d1b2cf86d446 --- /dev/null +++ b/be/test/storage/index/inverted/gram/gram_df_distribution_test.cpp @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// How much of the postings region pays for grams a query can never use? +// +// Two facts meet here. The section split shows the postings region holds most of the +// index -- 65% at the default scheme, 82% once max_gram is lowered. And the candidate +// ratio gate makes a gram with a high document frequency inert at query time: once its +// df crosses gram_index_max_candidate_ratio_bp of the segment, the node it appears in +// returns the full range rather than intersect, because reading a posting list that +// matches most of the segment costs more than the scan it saves. +// +// A gram that is inert at query time still costs its full posting list on disk and in +// S3. This measures the size of that overlap: the share of posting entries held by grams +// whose df is above a series of thresholds, and what dropping them would leave behind. +// +// Dropping is semantically safe in this direction. The index is a superset filter, so +// removing a conjunct from an AND widens the candidate set and can only add false +// positives, never false negatives -- and a query whose grams are all dropped degrades +// to a full scan, which is what the gate already does for it today. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/inverted/gram/gram_extractor.h" + +namespace doris::segment_v2::gram { +namespace { + +GramScheme Sparse(uint32_t min_len, uint32_t max_len, uint32_t density_permille) { + GramScheme s; + s.mode = GramMode::SPARSE; + s.min_len = min_len; + s.max_len = max_len; + s.density_permille = density_permille; + return s; +} + +std::vector LoadCorpus(const char* path, size_t max_rows) { + std::vector out; + std::ifstream in(path); + std::string line; + while (out.size() < max_rows && std::getline(in, line)) { + if (!line.empty()) { + out.push_back(line); + } + } + return out; +} + +struct Term { + uint64_t df = 0; // rows the gram appears in + uint64_t entries = 0; // posting entries it owns (one per row it appears in) +}; + +} // namespace + +// Reports, per scheme, how the postings region divides between grams a query can use and +// grams the gate already refuses to read. Assertions stay structural: the shape of the +// distribution is the finding, and pinning its values would only make the case brittle. +TEST(GramDfDistributionTest, HighDfGramsHoldMostOfThePostings) { + const char* corpus_path = std::getenv("GRAM_SECTION_CORPUS"); + if (corpus_path == nullptr) { + GTEST_SKIP() << "set GRAM_SECTION_CORPUS to a corpus file (one row per line) to run"; + } + const char* rows_env = std::getenv("GRAM_SECTION_ROWS"); + const size_t max_rows = rows_env ? static_cast(std::atoll(rows_env)) : 200000; + const std::vector corpus = LoadCorpus(corpus_path, max_rows); + ASSERT_FALSE(corpus.empty()) << "no rows read from " << corpus_path; + + const std::vector> schemes = { + {"d=0.25 max=16 (default)", Sparse(3, 16, 250)}, + {"d=0.25 max=4", Sparse(3, 4, 250)}, + {"d=0.10 max=16", Sparse(3, 16, 100)}, + {"d=0.05 max=16", Sparse(3, 16, 50)}, + {"d=0.05 max=4", Sparse(3, 4, 50)}, + }; + // The first is the shipped gate threshold (15 bp); the rest bracket it so the curve + // is visible rather than a single point. + const std::vector cutoff_bp = {15.0, 50.0, 100.0, 500.0, 1000.0}; + + printf("\ncorpus: %s, %zu rows\n", corpus_path, corpus.size()); + + for (const auto& [name, scheme] : schemes) { + GramExtractor ex(scheme); + std::unordered_map terms; + std::vector grams; + uint64_t total_entries = 0; + for (const std::string& row : corpus) { + // extract() is the index-side call and already deduplicates within the row, + // so each gram it returns is exactly one posting entry. + ex.extract(row, &grams); + for (std::string_view g : grams) { + Term& t = terms[std::string(g)]; + t.df++; + t.entries++; + total_entries++; + } + } + ASSERT_GT(total_entries, 0U) << name; + + printf("\n%s: %zu terms, %llu posting entries, %.1f entries/row\n", name, terms.size(), + static_cast(total_entries), + static_cast(total_entries) / static_cast(corpus.size())); + printf(" %-10s %10s %10s %10s %10s\n", "df cutoff", "terms>cut", "entries>cut", "entries%", + "terms%"); + for (double bp : cutoff_bp) { + const uint64_t cut = + static_cast(static_cast(corpus.size()) * bp / 10000.0); + uint64_t over_terms = 0, over_entries = 0; + for (const auto& [term, t] : terms) { + if (t.df > cut) { + over_terms++; + over_entries += t.entries; + } + } + printf(" df>%-7llu %10llu %10llu %9.1f%% %9.1f%%\n", + static_cast(cut), + static_cast(over_terms), + static_cast(over_entries), + 100.0 * static_cast(over_entries) / static_cast(total_entries), + 100.0 * static_cast(over_terms) / static_cast(terms.size())); + } + } +} + +} // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/gram_extractor_test.cpp b/be/test/storage/index/inverted/gram/gram_extractor_test.cpp new file mode 100644 index 00000000000000..65363ac6890362 --- /dev/null +++ b/be/test/storage/index/inverted/gram/gram_extractor_test.cpp @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/gram/gram_extractor.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "util/stopwatch.hpp" + +namespace doris::segment_v2::gram { + +static std::vector run(const GramScheme& s, std::string_view v) { + GramExtractor ex(s); + std::vector out; + ex.extract(v, &out); + return {out.begin(), out.end()}; +} + +TEST(GramExtractorTest, DenseAsciiTrigrams) { + GramScheme s; + s.mode = GramMode::DENSE; + s.min_len = 3; + EXPECT_EQ(run(s, "abcde"), (std::vector {"abc", "bcd", "cde"})); + EXPECT_EQ(run(s, "ab"), (std::vector {})); // shorter than n: nothing + EXPECT_EQ(run(s, "aaaa"), (std::vector {"aaa"})); // deduped within the row +} + +// Non-ASCII text is not indexed at all. +// +// It was, one term per code point, and that shape cannot be made to work. A term that is one +// character is too common to filter with, and every scheme that gives such a column a rarer +// term -- pairs, content-defined chunks of code points, pairs emitted only where both +// characters are common -- pays for it in vocabulary: measured on 500,000 rows of weibo the +// dictionary alone runs 5.5 MB to 22.8 MB against an 11.9 MB budget for the whole index, +// while one term per code point fits the dictionary and then needs its postings dropped to +// fit the budget, which leaves a CJK query with nothing to intersect. Six schemes, no +// configuration that is both small enough and useful. +// +// So the index declines the work rather than doing it badly. Correctness is unaffected: the +// extractor serves BOTH sides, so a literal's non-ASCII part contributes no gram to the query +// either, the candidate set is only ever widened, and the predicate re-checks every candidate +// row. A wholly non-ASCII pattern produces no grams at all and the query runs as a scan. +TEST(GramExtractorTest, NonAsciiTextProducesNoGrams) { + GramScheme s; + s.mode = GramMode::DENSE; + EXPECT_EQ(run(s, "手机微博"), (std::vector {})); + // The ASCII segments of a mixed string still index, and still do not span the CJK that + // separates them -- "abc" and "def" are two segments, not one six-byte run. + EXPECT_EQ(run(s, "abc手机def"), (std::vector {"abc", "def"})); + // An invalid UTF-8 byte is non-ASCII too, and is skipped rather than indexed as itself. + EXPECT_EQ(run(s, std::string("\xFF", 1)), (std::vector {})); + // A truncated sequence must not swallow the ASCII that follows it. + EXPECT_EQ(run(s, std::string("\xe6\x89", 2) + "abc"), (std::vector {"abc"})); +} + +TEST(GramExtractorTest, SparseIsLocalAndDeterministic) { + GramScheme s; // sparse p=0.25 n=3 L=16 + GramExtractor ex(s); + // Locality: the gram set of any substring is a subset of the gram set of the whole string + const std::string doc = "rpc error: code = Unavailable desc = error reading from server"; + std::vector all; + ex.extract(doc, &all); + std::set whole(all.begin(), all.end()); + for (size_t i = 0; i < doc.size(); i++) { + for (size_t len = 3; i + len <= doc.size(); len++) { + std::vector sub; + ex.extract(std::string_view(doc).substr(i, len), &sub); + for (auto g : sub) { + EXPECT_TRUE(whole.count(std::string(g))) << "gram '" << g << "' of substring [" << i + << "," << len << ") missing from whole"; + } + } + } + // Density: at 0.25 the gram count is well below the dense count (len-2) + EXPECT_LT(all.size(), (doc.size() - 2) / 2); +} + +TEST(GramExtractorTest, SparseGoldenFromPrototype) { + // The golden was generated by + // tools/regex-ngram-model/ngram_model.cpp --cdc --p 0.25 --maxlen 16 --n 3 + GramScheme s; + EXPECT_EQ(run(s, "rpc error: code = Unavailable"), + (std::vector {"or: co", "cod", "ode = U", " Unavai", "ailable"})); +} + +TEST(GramExtractorTest, LowerCaseFoldsBeforeBoundaryHash) { + GramScheme s; + s.lower_case = true; + EXPECT_EQ(run(s, "Code = Unavailable"), run(s, "code = unavailable")); +} + +// Covers Ruling R9: the extractor never emits a gram containing a NUL byte (0x00). +TEST(GramExtractorTest, NoGramContainsNulByte) { + // DENSE n=3: all three windows of "ab\0cd" (5 bytes) -- "ab\0", "b\0c", "\0cd" -- contain NUL + // and must all be skipped; the way window boundaries are computed is unchanged, they are just + // not emitted. + GramScheme dense; + dense.mode = GramMode::DENSE; + dense.min_len = 3; + EXPECT_EQ(run(dense, std::string("ab\0cd", 5)), (std::vector {})); + + // SPARSE: a NUL embedded in the middle of a longer string; candidate grams spanning the NUL + // are skipped, while the parts far from it still emit grams as usual (locality is + // unaffected). + GramScheme sparse; // SPARSE by default, p=0.25 n=3 max_len=16 + GramExtractor ex(sparse); + std::string doc = "rpc error: code = Unavailable desc = error reading from server"; + doc.insert(doc.begin() + 10, '\0'); + std::vector out; + ex.extract(doc, &out); + ASSERT_FALSE(out.empty()); + for (const auto& g : out) { + EXPECT_EQ(g.find('\0'), std::string_view::npos) << "gram '" << g << "' contains NUL"; + } +} + +namespace { + +// One row of `length` bytes drawn from the first `alphabet` printable ASCII characters. A narrow +// alphabet makes almost every window a repeat of an earlier one, a wide one makes almost every +// window unique, and alphabet=1 is the degenerate "one distinct gram, thousands of copies" case. +std::string random_ascii_row(uint64_t* seed, size_t length, unsigned alphabet) { + std::string row; + row.reserve(length); + for (size_t i = 0; i < length; i++) { + *seed = *seed * 6364136223846793005ULL + 1442695040888963407ULL; + row.push_back(static_cast(0x20 + (*seed >> 33) % alphabet)); + } + return row; +} + +} // namespace + +// Deduplication must keep exactly the first occurrence of every distinct gram and leave the +// survivors in their original relative order. That order is a contract rather than an +// implementation detail -- TOKENIZE() hands it to the user, and the ordered assertions above pin +// it -- so a future rewrite of the dedupe step (to shrink its transient memory, say) must not +// disturb it. DENSE mode makes the contract checkable against an independent reference: over a +// NUL-free pure-ASCII row it emits exactly the min_len-byte windows in ascending start order, so +// the expected output is that window sequence run through a plain first-appearance dedupe. +TEST(GramExtractorTest, DenseDedupeKeepsOrderOfFirstAppearance) { + const auto reference = [](const std::string& row, size_t min_len) { + std::vector expected; + std::unordered_set seen; + for (size_t i = 0; i + min_len <= row.size(); i++) { + const std::string_view window(row.data() + i, min_len); + if (seen.insert(window).second) { + expected.emplace_back(window); + } + } + return expected; + }; + uint64_t seed = 0x9e3779b97f4a7c15ULL; + for (unsigned alphabet : {1U, 2U, 5U, 26U, 95U}) { + for (size_t length : {size_t {1}, size_t {7}, size_t {64}, size_t {999}, size_t {4096}}) { + const std::string row = random_ascii_row(&seed, length, alphabet); + for (uint32_t min_len : {1U, 2U, 3U, 8U}) { + GramScheme scheme; + scheme.mode = GramMode::DENSE; + scheme.min_len = min_len; + SCOPED_TRACE("alphabet=" + std::to_string(alphabet) + " length=" + + std::to_string(length) + " min_len=" + std::to_string(min_len)); + EXPECT_EQ(run(scheme, row), reference(row, min_len)); + } + } + } +} + +// SPARSE grams have variable length, so there is no equally short closed form for the emitted +// sequence; what must still hold on that path is that one row's gram list never repeats a gram. +TEST(GramExtractorTest, SparseOutputNeverRepeatsAGram) { + uint64_t seed = 0x243f6a8885a308d3ULL; + for (unsigned alphabet : {1U, 2U, 5U, 95U}) { + for (size_t length : {size_t {3}, size_t {64}, size_t {4096}}) { + std::string row = random_ascii_row(&seed, length, alphabet); + row += "手机微博手机"; // a non-ASCII tail, which takes the 1-gram path + GramScheme scheme; // SPARSE by default, p=0.25 n=3 max_len=16 + SCOPED_TRACE("alphabet=" + std::to_string(alphabet) + + " length=" + std::to_string(length)); + const std::vector grams = run(scheme, row); + const std::set distinct(grams.begin(), grams.end()); + EXPECT_EQ(grams.size(), distinct.size()); + } + } +} + +TEST(GramExtractorTest, BoundaryHashVersionOneGolden) { + // Hash-version-1 goldens: FNV-1a over the 0/1 boundary results for all byte pairs in + // numeric order. Pin the complete rule, not merely the approximate boundary density. + struct Golden { + uint32_t density; + size_t count; + uint64_t digest; + }; + const Golden cases[] = {{.density = 1, .count = 76, .digest = 0xd82b5155d39e1de5ULL}, + {.density = 250, .count = 16488, .digest = 0x90bb2e083a7a9ec5ULL}, + {.density = 1000, .count = 65536, .digest = 0x2d438615f3572325ULL}}; + for (auto mode : {GramMode::DENSE, GramMode::SPARSE}) { + for (const auto& golden : cases) { + GramScheme scheme; + scheme.mode = mode; + scheme.density_permille = golden.density; + SCOPED_TRACE(scheme.cache_key()); + GramExtractor extractor(scheme); + size_t count = 0; + uint64_t digest = 14695981039346656037ULL; + for (unsigned pair = 0; pair < 65536; ++pair) { + const bool boundary = extractor.is_boundary(static_cast(pair >> 8), + static_cast(pair)); + count += boundary; + digest = (digest ^ boundary) * 1099511628211ULL; + } + EXPECT_EQ(count, golden.count); + EXPECT_EQ(digest, golden.digest); + } + } +} + +// A fresh tokenizer is created for each indexed value. Its setup must not dominate +// short-row extraction by an order of magnitude. Compare thread CPU time to exclude +// scheduling delays, and use a median with a deliberately generous local budget. +TEST(GramExtractorPerformanceTest, ShortRowsHaveBoundedConstructionOverhead) { + const std::string row = + "2026-09-05T12:45:30 INFO rpc request completed service=frontend " + "method=query user=demo elapsed_ms=17 status=OK remote=10.1.2.3"; + constexpr size_t iterations = 1024; + for (GramMode mode : {GramMode::DENSE, GramMode::SPARSE}) { + GramScheme scheme; + scheme.mode = mode; + const size_t expected_grams = mode == GramMode::DENSE ? 120 : 27; + std::array ratios {}; + for (double& ratio : ratios) { + size_t fresh_count = 0; + ThreadCpuStopWatch fresh_timer(true); + for (size_t i = 0; i < iterations; ++i) { + GramExtractor extractor(scheme); + std::vector grams; + extractor.extract(row, &grams); + fresh_count += grams.size(); + } + fresh_timer.stop(); + + GramExtractor extractor(scheme); + std::vector grams; + size_t reused_count = 0; + ThreadCpuStopWatch reused_timer(true); + for (size_t i = 0; i < iterations; ++i) { + extractor.extract(row, &grams); + reused_count += grams.size(); + } + reused_timer.stop(); + ASSERT_EQ(fresh_count, expected_grams * iterations); + ASSERT_EQ(reused_count, expected_grams * iterations); + ASSERT_GT(reused_timer.elapsed_time(), 0); + ratio = static_cast(fresh_timer.elapsed_time()) / + static_cast(reused_timer.elapsed_time()); + } + std::ranges::sort(ratios); + RecordProperty( + mode == GramMode::DENSE ? "dense_fresh_reused_ratio" : "sparse_fresh_reused_ratio", + std::to_string(ratios[1])); + EXPECT_LT(ratios[1], 10.0) + << "mode=" << static_cast(mode) << ", fresh/reused CPU ratio=" << ratios[1]; + } +} + +} // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/gram_literal_coverage_test.cpp b/be/test/storage/index/inverted/gram/gram_literal_coverage_test.cpp new file mode 100644 index 00000000000000..6d57d56cc4f6b6 --- /dev/null +++ b/be/test/storage/index/inverted/gram/gram_literal_coverage_test.cpp @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// How much of a query pattern's literal can actually be turned into grams? +// +// A row is tokenized as a whole, so its ASCII runs are long and content-defined +// boundaries are plentiful. A query only ever sees an isolated literal, and a gram it +// derives is usable only if the gram's whole extent lies inside that literal. Under +// SPARSE that needs either two boundaries inside the literal, or a full max_len window +// inside it -- so a literal shorter than max_len is queryable only by luck, and one +// that produces nothing degrades to ALL and the index goes unused. +// +// This measures that directly on the production extractor: for a grid of schemes, the +// share of literals of each length that yield at least one gram ("coverage"), plus the +// mean gram length, which is the other half of the trade -- shorter grams are easier to +// produce from a short literal and less selective once produced. + +#include + +#include +#include +#include +#include +#include +#include + +#include "storage/index/inverted/gram/gram_extractor.h" + +namespace doris::segment_v2::gram { +namespace { + +// Coverage is a property of the boundary distribution over adjacent byte pairs, so it +// depends on the corpus. GRAM_LITERAL_COVERAGE_CORPUS names a file of one row per line +// to measure against real text; without it the built-in rows below stand in. They are +// deterministic and in the shape of the benchmarked data -- request lines, service log +// lines, structured agent observations, all ASCII -- but they are templated, so read the +// built-in run as the shape of the curve and a real corpus for its values. The assertions +// hold either way: they test monotonicity and DENSE's guarantee, not particular numbers. +std::vector BuiltinCorpus() { + std::vector rows; + const char* verbs[] = {"GET", "POST", "HEAD"}; + const char* dirs[] = {"images", "english", "french", "history", "venues"}; + const char* exts[] = {"gif", "jpg", "html", "htm", "txt"}; + for (int i = 0; i < 400; i++) { + rows.push_back(std::string(verbs[i % 3]) + " /" + dirs[i % 5] + "/nav_" + + std::to_string(i * 7919 % 100000) + "." + exts[i % 5] + " HTTP/1.0"); + rows.push_back( + "rpc error: code = Unavailable desc = error reading from server: " + "context deadline exceeded, trace.rb:" + + std::to_string(i % 500) + ":in block"); + rows.push_back("{\"cwd\":\"/workspace/tenant_" + std::to_string(i % 32) + + "/app_013\",\"target\":\"/workspace/demo/app_013/trace_" + + std::to_string(i * 31 % 999983) + "/mcp__matrix__bash_exec_" + + std::to_string(i % 8) + ".md\"}"); + } + return rows; +} + +const std::vector& Corpus() { + static const std::vector rows = [] { + const char* path = std::getenv("GRAM_LITERAL_COVERAGE_CORPUS"); + if (path == nullptr) { + return BuiltinCorpus(); + } + const char* rows_env = std::getenv("GRAM_LITERAL_COVERAGE_ROWS"); + const size_t max_rows = rows_env ? static_cast(std::atoll(rows_env)) : 20000; + std::vector out; + std::ifstream in(path); + std::string line; + while (out.size() < max_rows && std::getline(in, line)) { + if (!line.empty()) { + out.push_back(line); + } + } + return out.empty() ? BuiltinCorpus() : out; + }(); + return rows; +} + +struct Coverage { + double covered = 0.0; // share of literals yielding >= 1 gram + double mean_len = 0.0; // mean length of the grams produced +}; + +// Every substring of the given length, taken from the corpus, run through the real +// extractor's literal path -- the same call the compiler makes. +Coverage Measure(const GramScheme& s, size_t lit_len) { + GramExtractor ex(s); + size_t total = 0, hit = 0, gram_bytes = 0, gram_count = 0; + std::vector grams; + for (const std::string& row : Corpus()) { + if (row.size() < lit_len) { + continue; + } + for (size_t off = 0; off + lit_len <= row.size(); off += 3) { + ex.grams_of_literal(std::string_view(row).substr(off, lit_len), &grams); + total++; + if (!grams.empty()) { + hit++; + for (const std::string& g : grams) { + gram_bytes += g.size(); + } + gram_count += grams.size(); + } + } + } + Coverage c; + c.covered = total ? static_cast(hit) / static_cast(total) : 0.0; + c.mean_len = + gram_count ? static_cast(gram_bytes) / static_cast(gram_count) : 0.0; + return c; +} + +GramScheme Sparse(uint32_t min_len, uint32_t max_len, uint32_t density_permille) { + GramScheme s; + s.mode = GramMode::SPARSE; + s.min_len = min_len; + s.max_len = max_len; + s.density_permille = density_permille; + return s; +} + +} // namespace + +// Reports the coverage grid. It asserts only the two properties the design rests on, +// so it cannot fail for a change in corpus or hash: coverage must not fall as the +// literal grows, and DENSE must cover every literal at or above min_len. +TEST(GramLiteralCoverageTest, ShortLiteralsAreOftenUnqueryableUnderSparse) { + struct Cfg { + const char* name; + GramScheme scheme; + }; + // The two levers are crossed rather than swept one at a time. Lowering density is + // what shrinks the index, and it costs coverage. Lowering max_gram shrinks the + // dictionary at almost no cost to the postings, and it should *raise* coverage: a + // literal too short to hold two boundaries can still hold one fixed window once the + // window is short enough. Whether the second offsets the first is exactly the + // question a one-lever sweep cannot answer. + const std::vector cfgs = { + {"sparse d=050 max=16", Sparse(3, 16, 50)}, + {"sparse d=050 max=8", Sparse(3, 8, 50)}, + {"sparse d=050 max=6", Sparse(3, 6, 50)}, + {"sparse d=050 max=4", Sparse(3, 4, 50)}, + {"sparse d=100 max=16", Sparse(3, 16, 100)}, + {"sparse d=100 max=6", Sparse(3, 6, 100)}, + {"sparse d=100 max=4", Sparse(3, 4, 100)}, + {"sparse d=150 max=16", Sparse(3, 16, 150)}, + {"sparse d=250 max=16 (default)", Sparse(3, 16, 250)}, + {"sparse d=250 max=8", Sparse(3, 8, 250)}, + {"sparse d=250 max=6", Sparse(3, 6, 250)}, + {"sparse d=250 max=4", Sparse(3, 4, 250)}, + {"sparse d=500 max=16", Sparse(3, 16, 500)}, + {"sparse d=500 max=6", Sparse(3, 6, 500)}, + }; + const std::vector lens = {3, 4, 6, 8, 10, 12, 16, 20, 24}; + + for (const Cfg& cfg : cfgs) { + std::string line = std::string(cfg.name) + " |"; + std::string mean = std::string(cfg.name) + " | mean gram len:"; + double prev = -1.0; + for (size_t l : lens) { + const Coverage c = Measure(cfg.scheme, l); + char buf[64]; + snprintf(buf, sizeof(buf), " L%zu=%.0f%%", l, c.covered * 100.0); + line += buf; + snprintf(buf, sizeof(buf), " L%zu=%.1f", l, c.mean_len); + mean += buf; + // Coverage is monotone in literal length: a longer literal contains every + // window the shorter one did. + EXPECT_GE(c.covered + 1e-9, prev) << cfg.name << " length " << l; + prev = c.covered; + } + std::cout << line << "\n" << mean << "\n"; + } + + GramScheme dense; + dense.mode = GramMode::DENSE; + dense.min_len = 3; + std::string dline = "dense min=3 |"; + for (size_t l : lens) { + const Coverage c = Measure(dense, l); + char buf[64]; + snprintf(buf, sizeof(buf), " L%zu=%.0f%%", l, c.covered * 100.0); + dline += buf; + EXPECT_DOUBLE_EQ(c.covered, 1.0) << "dense must cover every literal >= min_len"; + } + std::cout << dline << "\n"; +} + +} // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/gram_query_test.cpp b/be/test/storage/index/inverted/gram/gram_query_test.cpp new file mode 100644 index 00000000000000..79628e84d5106a --- /dev/null +++ b/be/test/storage/index/inverted/gram/gram_query_test.cpp @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/gram/gram_query.h" + +#include + +namespace doris::segment_v2::gram { + +TEST(GramQueryTest, AndOrShortCircuit) { + EXPECT_TRUE(GramQuery::and_(GramQuery::all(), GramQuery::of_gram("abc")).op == + GramQuery::Op::AND); + EXPECT_TRUE(GramQuery::and_(GramQuery::none(), GramQuery::of_gram("abc")).is_none()); + EXPECT_TRUE(GramQuery::or_(GramQuery::all(), GramQuery::of_gram("abc")).is_all()); + EXPECT_EQ(GramQuery::or_(GramQuery::none(), GramQuery::of_gram("abc")).to_debug_string(), + "(\"abc\")"); +} + +TEST(GramQueryTest, FlattenDedupeAbsorb) { + auto q = GramQuery::and_(GramQuery::and_(GramQuery::of_gram("abc"), GramQuery::of_gram("bcd")), + GramQuery::of_gram("abc")); + EXPECT_EQ(q.op, GramQuery::Op::AND); + EXPECT_EQ(q.grams.size(), 2U); + // abc is already in the AND, so the child OR(abc|xyz) is always true and gets absorbed + auto r = GramQuery::and_(q, + GramQuery::or_(GramQuery::of_gram("abc"), GramQuery::of_gram("xyz"))); + EXPECT_EQ(r.leaf_count(), 2U); + // The gram "x" is already in the OR, so a child AND containing "x" is absorbed (the other + // direction of absorption): x | (x & y) -> x + auto t = GramQuery::or_(GramQuery::of_gram("x"), + GramQuery::and_(GramQuery::of_gram("x"), GramQuery::of_gram("y"))); + EXPECT_EQ(t.to_debug_string(), "(\"x\")"); + // Inside an OR, a subset AND absorbs its superset: (a&b) | (a&b&c) -> (a&b) + auto s = GramQuery::or_( + GramQuery::and_(GramQuery::of_gram("a"), GramQuery::of_gram("b")), + GramQuery::and_(GramQuery::and_(GramQuery::of_gram("a"), GramQuery::of_gram("b")), + GramQuery::of_gram("c"))); + EXPECT_EQ(s.to_debug_string(), "(\"a\" & \"b\")"); +} + +TEST(GramQueryTest, StructuralKeyIgnoresOperandOrder) { + auto left = GramQuery::or_(GramQuery::of_gram("a"), GramQuery::of_gram("b")); + auto right = GramQuery::or_(GramQuery::of_gram("c"), GramQuery::of_gram("d")); + auto query = GramQuery::and_(left, right); + EXPECT_EQ(query.structural_key(), GramQuery::and_(right, left).structural_key()); + EXPECT_EQ(left.structural_key(), + GramQuery::or_(GramQuery::of_gram("b"), GramQuery::of_gram("a")).structural_key()); + auto deduplicated = GramQuery::and_(query, left); + EXPECT_EQ(deduplicated.leaf_count(), 4U); + EXPECT_EQ(deduplicated.structural_key(), query.structural_key()); +} + +TEST(GramQueryTest, StructuralKeyDistinguishesGramBytesFromOperators) { + EXPECT_NE(GramQuery::of_gram("a,b").structural_key(), + GramQuery::and_(GramQuery::of_gram("a"), GramQuery::of_gram("b")).structural_key()); + EXPECT_NE(GramQuery::of_gram("*").structural_key(), GramQuery::all().structural_key()); + EXPECT_NE(GramQuery::of_gram("!").structural_key(), GramQuery::none().structural_key()); + EXPECT_NE(GramQuery::all().structural_key(), GramQuery::none().structural_key()); + EXPECT_NE(GramQuery::of_gram(std::string("a\0b", 3)).structural_key(), + GramQuery::of_gram("a").structural_key()); +} + +} // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/gram_scheme_test.cpp b/be/test/storage/index/inverted/gram/gram_scheme_test.cpp new file mode 100644 index 00000000000000..75d732310990bd --- /dev/null +++ b/be/test/storage/index/inverted/gram/gram_scheme_test.cpp @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/gram/gram_scheme.h" + +#include + +#include +#include +#include + +namespace doris::segment_v2::gram { + +TEST(GramSchemeTest, DefaultsAndRoundTrip) { + GramScheme s; + EXPECT_EQ(s.mode, GramMode::SPARSE); + EXPECT_EQ(s.min_len, 3U); + EXPECT_EQ(s.max_len, 16U); + EXPECT_EQ(s.density_permille, 250U); + GramScheme back; + ASSERT_TRUE(GramScheme::from_properties(s.to_properties(), &back).ok()); + EXPECT_TRUE(s == back); + EXPECT_EQ(s.cache_key(), "gram:v1:sparse:3:16:250:lc0"); +} + +TEST(GramSchemeTest, ParsesTokenizerProperties) { + std::map props = {{"mode", "dense"}, + {"min_gram", "4"}, + {"max_gram", "24"}, + {"density", "0.33"}, + {"lower_case", "true"}}; + GramScheme s; + ASSERT_TRUE(GramScheme::from_properties(props, &s).ok()); + EXPECT_EQ(s.mode, GramMode::DENSE); + EXPECT_EQ(s.min_len, 4U); + EXPECT_EQ(s.max_len, 24U); + EXPECT_EQ(s.density_permille, 330U); + EXPECT_TRUE(s.lower_case); +} + +TEST(GramSchemeTest, LowerCaseAcceptsBooleanSynonyms) { + GramScheme s; + ASSERT_TRUE(GramScheme::from_properties({{"lower_case", "true"}}, &s).ok()); + EXPECT_TRUE(s.lower_case); + ASSERT_TRUE(GramScheme::from_properties({{"lower_case", "1"}}, &s).ok()); + EXPECT_TRUE(s.lower_case); + ASSERT_TRUE(GramScheme::from_properties({{"lower_case", "false"}}, &s).ok()); + EXPECT_FALSE(s.lower_case); + ASSERT_TRUE(GramScheme::from_properties({{"lower_case", "0"}}, &s).ok()); + EXPECT_FALSE(s.lower_case); +} + +TEST(GramSchemeTest, RejectsInvalid) { + GramScheme s; + EXPECT_FALSE(GramScheme::from_properties({{"mode", "fuzzy"}}, &s).ok()); + EXPECT_FALSE(GramScheme::from_properties({{"min_gram", "0"}}, &s).ok()); + EXPECT_FALSE(GramScheme::from_properties({{"min_gram", "8"}, {"max_gram", "4"}}, &s).ok()); + EXPECT_FALSE(GramScheme::from_properties({{"density", "0"}}, &s).ok()); + EXPECT_FALSE(GramScheme::from_properties({{"density", "1.5"}}, &s).ok()); + EXPECT_FALSE(GramScheme::from_properties({{"lower_case", "yes"}}, &s).ok()); +} + +// A gram scheme has no high-frequency-gram pruning threshold: no reader consumes one, so the +// parameter is not part of the scheme identity, is not written back to the properties and does +// not reach the cache key. Anything named after it is just an unrecognised property. +TEST(GramSchemeTest, NoStopGramDfConcept) { + GramScheme parsed; + ASSERT_TRUE(GramScheme::from_properties({{"stop_gram_df", "0.25"}}, &parsed).ok()); + EXPECT_TRUE(parsed == GramScheme {}); + + const auto props = GramScheme {}.to_properties(); + EXPECT_EQ(props.find("stop_gram_df"), props.end()); + EXPECT_EQ(GramScheme {}.cache_key().find("stop"), std::string::npos); +} + +TEST(GramSchemeTest, ParsesPortableDecimalProperties) { + const std::pair cases[] = {{"0.25", 250}, {".25", 250}, {"1.", 1000}, + {"+0.25", 250}, {"2.5e-1", 250}, {"0.001", 1}, + {"1", 1000}}; + for (const auto& [value, expected] : cases) { + SCOPED_TRACE(std::string("density=") + value); + GramScheme scheme; + ASSERT_TRUE(GramScheme::from_properties({{"density", value}}, &scheme).ok()); + EXPECT_EQ(scheme.density_permille, expected); + } + for (const auto* value : {"0.25f", "0.25D", "0.25 ", "0.25\t"}) { + GramScheme scheme; + EXPECT_FALSE(GramScheme::from_properties({{"density", value}}, &scheme).ok()); + } +} + +} // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/regex_ast_test.cpp b/be/test/storage/index/inverted/gram/regex_ast_test.cpp new file mode 100644 index 00000000000000..e56c893ce5b3d3 --- /dev/null +++ b/be/test/storage/index/inverted/gram/regex_ast_test.cpp @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/gram/regex_ast.h" + +#include + +namespace doris::segment_v2::gram { + +static std::string dump(const RegexNode* n) { + using T = RegexNode::Type; + switch (n->type) { + case T::EMPTY: + return "e"; + case T::LIT: + return "'" + n->lit + "'"; + case T::CLASS: { + if (n->big_class) { + return "[big]"; + } + std::string s = "["; + for (const auto& c : n->cls) { + s += c; + } + return s + "]"; + } + case T::ANY: + return "."; + case T::CAT: { + std::string s = "cat("; + for (const auto& k : n->kids) { + s += dump(k.get()) + ","; + } + return s + ")"; + } + case T::ALT: { + std::string s = "alt("; + for (const auto& k : n->kids) { + s += dump(k.get()) + ","; + } + return s + ")"; + } + case T::STAR: + return "star(" + dump(n->kids[0].get()) + ")"; + case T::PLUS: + return "plus(" + dump(n->kids[0].get()) + ")"; + case T::QUEST: + return "quest(" + dump(n->kids[0].get()) + ")"; + case T::REPEAT: + return "rep(" + dump(n->kids[0].get()) + "," + std::to_string(n->rmin) + "," + + std::to_string(n->rmax) + ")"; + } + return "?"; +} + +static std::string parse_dump(const std::string& re) { + std::unique_ptr root; + bool icase = false; + Status st = parse_regex(re, &root, &icase); + if (!st.ok()) { + return "ERR"; + } + return dump(root.get()); +} + +TEST(RegexAstTest, Basics) { + EXPECT_EQ(parse_dump("abc"), "cat('a','b','c',)"); + EXPECT_EQ(parse_dump("a|bc"), "alt(cat('a',),cat('b','c',),)"); + EXPECT_EQ(parse_dump("a.*b"), "cat('a',star(.),'b',)"); + EXPECT_EQ(parse_dump("(ab)+c?"), "cat(plus(cat('a','b',)),quest('c'),)"); + EXPECT_EQ(parse_dump("x{2,3}"), "cat(rep('x',2,3),)"); + EXPECT_EQ(parse_dump("^\\d{3}-\\d{4}$"), "cat(e,rep([big],3,3),'-',rep([big],4,4),e,)"); +} + +TEST(RegexAstTest, ClassesAndEscapes) { + EXPECT_EQ(parse_dump("[ab]"), "cat([ab],)"); + EXPECT_EQ(parse_dump("[a-z]"), "cat([big],)"); + EXPECT_EQ(parse_dump("[^a]"), "cat([big],)"); + EXPECT_EQ(parse_dump("\\.\\Qa.b\\E"), "cat('.','a','.','b',)"); + EXPECT_EQ(parse_dump("\\x41"), "cat('A',)"); + EXPECT_EQ(parse_dump("手机"), "cat('手','机',)"); +} + +TEST(RegexAstTest, FlagsAndErrors) { + std::unique_ptr root; + bool icase = false; + ASSERT_TRUE(parse_regex("(?i)ab", &root, &icase).ok()); + EXPECT_TRUE(icase); + EXPECT_EQ(dump(root.get()), "cat([Aa],[Bb],)"); + EXPECT_EQ(parse_dump("(ab"), "ERR"); + EXPECT_EQ(parse_dump("*a"), "ERR"); + EXPECT_EQ(parse_dump("[ab"), "ERR"); + EXPECT_EQ(parse_dump("a\\"), "ERR"); +} + +// Covers Important 1 of fix round 1: a bare '\' at the end of a class (`[a\` / `[a-\`) must not +// read past the end of the input. A non-NUL-terminated string_view (a prefix cut out of a longer +// buffer, rather than the NUL-terminated std::string the prototype always passed) verifies that +// the parser relies solely on the string_view's own length to detect the end of input, not on the +// string being NUL-terminated. +TEST(RegexAstTest, ClassTrailingBackslashAtEof) { + std::string buf1 = "[a\\XYZ"; + std::string_view view1 = std::string_view(buf1).substr(0, 3); // "[a\" + std::unique_ptr root1; + bool icase1 = false; + EXPECT_FALSE(parse_regex(view1, &root1, &icase1).ok()); + + std::string buf2 = "[a-\\XYZ"; + std::string_view view2 = std::string_view(buf2).substr(0, 4); // "[a-\" + std::unique_ptr root2; + bool icase2 = false; + EXPECT_FALSE(parse_regex(view2, &root2, &icase2).ok()); +} + +// Covers Important 2 of fix round 1: cls must be empty whenever big_class=true (the invariant in +// the RegexNode comment). Under (?i) the small-class expansion can double <= 4 original code +// points to more than 4 items and degrade to a large class, and cls must then be cleared too; +// below the cap it must still be an ordinary enumerable small class. +TEST(RegexAstTest, IcaseClassBigClassInvariant) { + std::unique_ptr root; + bool icase = false; + ASSERT_TRUE(parse_regex("(?i)[abc]", &root, &icase).ok()); + ASSERT_EQ(root->kids.size(), 1U); + const RegexNode* cls_node = root->kids[0].get(); + EXPECT_EQ(cls_node->type, RegexNode::Type::CLASS); + EXPECT_TRUE(cls_node->big_class); + EXPECT_TRUE(cls_node->cls.empty()); + + std::unique_ptr root2; + bool icase2 = false; + ASSERT_TRUE(parse_regex("(?i)[ab]", &root2, &icase2).ok()); + ASSERT_EQ(root2->kids.size(), 1U); + const RegexNode* cls_node2 = root2->kids[0].get(); + EXPECT_EQ(cls_node2->type, RegexNode::Type::CLASS); + EXPECT_FALSE(cls_node2->big_class); + EXPECT_EQ(cls_node2->cls, (std::vector {"A", "B", "a", "b"})); +} + +// Regression test for the kMaxNestingDepth (64) group nesting cap: both a run of bare (unclosed) +// opening parentheses and a balanced deep nesting must error out past the cap, rather than keep +// recursing until the stack overflows or a malformed tree is produced. +TEST(RegexAstTest, NestingDepthCapped) { + EXPECT_EQ(parse_dump(std::string(70, '(')), "ERR"); + EXPECT_EQ(parse_dump(std::string(70, '(') + "a" + std::string(70, ')')), "ERR"); +} + +// Covers Ruling R12: `\x` escape hardening. The bare `\xHH` form must have exactly two hex +// digits, and fewer than two (end of string, or a non-hex character) is always an error, matching +// RE2's rejection of `\x4`; the `\x{...}` form must have valid hex content and be closed, or it +// is an error as well. +TEST(RegexAstTest, HexEscapeHardening) { + EXPECT_EQ(parse_dump("\\x4"), "ERR"); + EXPECT_EQ(parse_dump("\\x"), "ERR"); + EXPECT_EQ(parse_dump("\\x4g"), "ERR"); + EXPECT_EQ(parse_dump("\\x41"), "cat('A',)"); + EXPECT_EQ(parse_dump("\\x{41}"), "cat('A',)"); + EXPECT_EQ(parse_dump("[\\x4]"), "ERR"); + EXPECT_EQ(parse_dump("\\x{4"), "ERR"); +} + +} // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/regex_gram_compiler_test.cpp b/be/test/storage/index/inverted/gram/regex_gram_compiler_test.cpp new file mode 100644 index 00000000000000..2c995efdbf33e4 --- /dev/null +++ b/be/test/storage/index/inverted/gram/regex_gram_compiler_test.cpp @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/gram/regex_gram_compiler.h" + +#include + +#include +#include +#include + +namespace doris::segment_v2::gram { + +// Dense golden: byte-for-byte identical to the output of the prototype's +// `ngram_model_check --n 3 --explain ` (for 3-byte grams the numeric order matches the +// lexicographic one, so the prototype's hash-order printout is already lexicographic). +static std::string dense(const std::string& re) { + GramScheme s; + s.mode = GramMode::DENSE; + s.min_len = 3; + RegexGramCompiler c(s); + GramQuery q; + EXPECT_TRUE(c.compile_regexp(re, &q).ok()); + return q.to_debug_string(); +} + +// Sparse golden: corresponds to the prototype's +// `--n 3 --cdc --p 0.25 --maxlen 24 --explain `; the gram sets are the same, but GramQuery +// emits them in lexicographic order (the prototype prints them in hash order). +static std::string sparse(const std::string& re) { + GramScheme s; + s.max_len = 24; + RegexGramCompiler c(s); + GramQuery q; + EXPECT_TRUE(c.compile_regexp(re, &q).ok()); + return q.to_debug_string(); +} + +TEST(RegexGramCompilerTest, DenseGolden) { + EXPECT_EQ(dense("abc"), "(\"abc\")"); + EXPECT_EQ(dense("a.*b"), "ALL"); + EXPECT_EQ(dense("\\d{3}-\\d{4}"), "ALL"); + EXPECT_EQ(dense("hello|world"), + "((\"ell\" & \"hel\" & \"llo\") | (\"orl\" & \"rld\" & \"wor\"))"); + EXPECT_EQ( + dense("(foo|bar)baz"), + "((\"arb\" & \"bar\" & \"baz\" & \"rba\") | (\"baz\" & \"foo\" & \"oba\" & \"oob\"))"); + EXPECT_EQ(dense("conn(ection)? re(set|fused)"), + "(\" re\" & \"con\" & \"onn\" & ((\"efu\" & \"fus\" & \"ref\" & \"sed\" & \"use\") | " + "(\"ese\" & \"res\" & \"set\")))"); + EXPECT_EQ(dense("GET|POST"), "(\"GET\" | (\"OST\" & \"POS\"))"); + EXPECT_EQ(dense("a(b|cd)e"), "(\"abe\" | (\"acd\" & \"cde\"))"); + EXPECT_EQ(dense("[ab]cd"), "(\"acd\" | \"bcd\")"); + EXPECT_EQ(dense("(abc){2}"), "(\"abc\" & \"bca\" & \"cab\")"); + EXPECT_EQ(dense("error.*timeout"), + "(\"eou\" & \"err\" & \"ime\" & \"meo\" & \"out\" & \"ror\" & \"rro\" & \"tim\")"); +} + +TEST(RegexGramCompilerTest, SparseGolden) { + EXPECT_EQ(sparse("rpc error: code = Unavailable"), + "(\" Unavai\" & \"ailable\" & \"cod\" & \"ode = U\" & \"or: co\")"); + EXPECT_EQ(sparse("error.*timeout"), "(\"timeo\")"); + EXPECT_EQ(sparse("GET|POST"), "ALL"); +} + +TEST(RegexGramCompilerTest, ParseErrorIsAll) { + GramScheme s; + RegexGramCompiler c(s); + GramQuery q; + ASSERT_TRUE(c.compile_regexp("(ab", &q).ok()); + EXPECT_TRUE(q.is_all()); +} + +TEST(RegexGramCompilerTest, CaseInsensitiveWithAndWithoutFolding) { + GramScheme s; + s.mode = GramMode::DENSE; + RegexGramCompiler c(s); + GramQuery q; + ASSERT_TRUE(c.compile_regexp("(?i)abcd", &q).ok()); + EXPECT_EQ(q.to_debug_string(), + "((\"ABC\" | \"ABc\" | \"AbC\" | \"Abc\" | \"aBC\" | \"aBc\" | \"abC\" | \"abc\") & " + "(\"BCD\" | \"BCd\" | \"BcD\" | \"Bcd\" | \"bCD\" | \"bCd\" | \"bcD\" | \"bcd\"))"); + s.lower_case = true; + RegexGramCompiler c2(s); + ASSERT_TRUE(c2.compile_regexp("(?i)ABCD", &q).ok()); + EXPECT_EQ(q.to_debug_string(), "(\"abc\" & \"bcd\")"); +} + +TEST(RegexGramCompilerTest, Like) { + GramScheme s; + s.mode = GramMode::DENSE; + RegexGramCompiler c(s); + GramQuery q; + ASSERT_TRUE(c.compile_like("%abcd%ef_gh%", &q).ok()); + EXPECT_EQ(q.to_debug_string(), "(\"abc\" & \"bcd\")"); // "ef" and "gh" are shorter than 3 + ASSERT_TRUE(c.compile_like("abc\\%def", &q).ok()); // an escaped % is a literal + EXPECT_EQ(q.to_debug_string(), "(\"%de\" & \"abc\" & \"bc%\" & \"c%d\" & \"def\")"); + ASSERT_TRUE(c.compile_like("%", &q).ok()); + EXPECT_TRUE(q.is_all()); +} + +// Ruling R10: the LIKE escape character is fixed to `\`, and only `\%`, `\_` and `\\` are real +// escapes; for `\x` (x none of those three) it is unclear whether the engine keeps the backslash +// itself (two bytes "\x" in the row) or drops it (just "x" in the row, the old implementation's +// wrong assumption). All that is certainly adjacent under both readings is "x and whatever +// follows it", so the segment must be cut at the backslash: the literal before it (such as "abc") +// may not be merged with x, while x may still form a new segment with what comes after it (such +// as "def"). +TEST(RegexGramCompilerTest, LikeEscapeConservative) { + GramScheme s; + s.mode = GramMode::DENSE; + RegexGramCompiler c(s); + GramQuery q; + + // `\d` is not a known escape: cut between "abc" and "d"; "d" is still adjacent to the "ef" + // after it, forming the new segment "def". Each segment yields exactly one 3-gram. + ASSERT_TRUE(c.compile_like("abc\\def", &q).ok()); + EXPECT_EQ(q.to_debug_string(), "(\"abc\" & \"def\")"); + + // An existing golden that must not change: `\%` is a known escape, so "%" joins the previous + // segment. + ASSERT_TRUE(c.compile_like("abc\\%def", &q).ok()); + EXPECT_EQ(q.to_debug_string(), "(\"%de\" & \"abc\" & \"bc%\" & \"c%d\" & \"def\")"); + + // `\\` escapes to one literal backslash, forming the contiguous literal segment "ab\cd" + // (5 bytes: a b \ c d) together with its neighbours. The resulting grams contain 0x5C, which + // sorts before the letters; comparing to_debug_string directly would need two layers of + // manual escaping, so we compare the sorted q.grams instead to avoid escaping mistakes. + ASSERT_TRUE(c.compile_like("ab\\\\cd", &q).ok()); + ASSERT_EQ(q.op, GramQuery::Op::AND); + EXPECT_TRUE(q.subs.empty()); + std::vector grams = q.grams; + std::sort(grams.begin(), grams.end()); + EXPECT_EQ(grams, std::vector({"\\cd", "ab\\", "b\\c"})); + + // A lone trailing backslash: nothing to escape, so the segment is cut and it is ignored. + ASSERT_TRUE(c.compile_like("abc\\", &q).ok()); + EXPECT_EQ(q.to_debug_string(), "(\"abc\")"); + + // The escaped % yields a 1-byte literal segment, too short for one gram (n=3) -> ALL. + ASSERT_TRUE(c.compile_like("%\\%", &q).ok()); + EXPECT_TRUE(q.is_all()); +} + +} // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/regex_gram_fuzz_test.cpp b/be/test/storage/index/inverted/gram/regex_gram_fuzz_test.cpp new file mode 100644 index 00000000000000..12c646646d29b5 --- /dev/null +++ b/be/test/storage/index/inverted/gram/regex_gram_fuzz_test.cpp @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include + +#include "storage/index/inverted/gram/gram_extractor.h" +#include "storage/index/inverted/gram/regex_gram_compiler.h" + +namespace doris::segment_v2::gram { + +namespace { + +// Evaluate a query tree over the gram set of one row. It deliberately reuses none of the +// simplification logic of GramQuery::and_/or_ and re-implements the evaluation from scratch by Op +// semantics, which is what makes this function a "truth interpreter" independent of the compiler +// under test: calling the code under test here would reduce the differential test to verifying +// the code under test with itself. +bool eval(const GramQuery& q, const std::set& grams) { + switch (q.op) { + case GramQuery::Op::ALL: + return true; + case GramQuery::Op::NONE: + return false; + case GramQuery::Op::AND: + return std::ranges::all_of(q.grams, [&](const auto& g) { return grams.contains(g); }) && + std::ranges::all_of(q.subs, [&](const auto& s) { return eval(s, grams); }); + case GramQuery::Op::OR: + return std::ranges::any_of(q.grams, [&](const auto& g) { return grams.contains(g); }) || + std::ranges::any_of(q.subs, [&](const auto& s) { return eval(s, grams); }); + } + return true; +} + +// The atom vocabulary shared by random regexes and random rows: the first 10 entries are "natural +// language" words/tokens (random_row draws from that range too when assembling a row), the last +// 10 are regex metacharacters/classes used only by random_regex. +const char* kAtoms[] = {"error", "code", "Unavailable", "timeout", "user_id=", "10.68.", "GET", + "POST", "手机", "微博", "[0-9]", "[a-z]", "\\d", ".", + "a", "b", "c", " ", "=", ":"}; + +// Generate a random regex within the RE2 syntax subset: 1 to 3 concatenated atoms, each with +// roughly even odds of being wrapped in one of an alternation (a|b), ?, +, *, {1,3}; depth caps +// how deep alternation branches recurse, so the regex cannot explode in size at random (a second +// safety net alongside RegexGramCompiler's own depth protection). +std::string random_regex(std::mt19937& rng, int depth) { + std::uniform_int_distribution pick(0, 19), shape(0, 9); + std::string s; + int parts = 1 + rng() % 3; + for (int i = 0; i < parts; i++) { + int sh = shape(rng); + std::string atom = kAtoms[pick(rng)]; + if (depth > 0 && sh == 0) { + atom = "(" + random_regex(rng, depth - 1) + "|" + random_regex(rng, depth - 1) + ")"; + } + if (sh == 1) { + atom = "(" + atom + ")?"; + } + if (sh == 2) { + atom = "(" + atom + ")+"; + } + if (sh == 3) { + atom = "(" + atom + ")*"; + } + if (sh == 4) { + atom = "(" + atom + "){1,3}"; + } + s += atom; + } + return s; +} + +// Generate one random row of text: pick one of 8 preset templates and, with probability 1/3, +// append a "natural language" atom. The templates and atoms (two CJK templates included) are all +// valid UTF-8 literals, so this function always produces a valid UTF-8 row -- RE2's behaviour of +// refusing to match an invalid UTF-8 row is never triggered by this test. +std::string random_row(std::mt19937& rng) { + static const char* kRows[] = {"rpc error: code = Unavailable desc = timeout", + "user_id=abc GET /images/x.gif", + "手机微博 POST 10.68.3.18:8080 error", + "Convert conversion successful", + "", + "aaa bbb ccc", + "code=Unavailable", + "timeout after error error error"}; + std::string r = kRows[rng() % 8]; + if (rng() % 3 == 0) { + r += kAtoms[rng() % 10]; + } + return r; +} + +// Run "compile + a differential check over 20 rows" for one regex; extracted from TestBody to +// keep its cognitive complexity down (clang-tidy readability-function-cognitive-complexity, +// threshold 50). gtest allows ASSERT_* inside an ordinary void function: on failure the return +// inside the macro only leaves this function and does not terminate the caller's loop, so the +// caller must still check HasFatalFailure() after every call to preserve the original "stop +// everything as soon as a false negative appears" behaviour. +void check_regex(GramMode mode, bool lc, GramExtractor& ex, RegexGramCompiler& comp, + std::mt19937& rng, const std::string& re, int* compiled, int* indexable) { + RE2 rx(re, RE2::Quiet); + if (!rx.ok()) { + return; + } + GramQuery q; + ASSERT_TRUE(comp.compile_regexp(re, &q).ok()); + (*compiled)++; + if (!q.is_all()) { + (*indexable)++; + } + for (int r = 0; r < 20; r++) { + std::string row = random_row(rng); + bool truth = RE2::PartialMatch(row, rx); + std::vector g; + ex.extract(row, &g); + std::set grams(g.begin(), g.end()); + bool cand = eval(q, grams); + ASSERT_TRUE(!truth || cand) + << "FALSE NEGATIVE mode=" << (int)mode << " lc=" << lc << " re=" << re + << " row=" << row << " q=" << q.to_debug_string(); + } +} + +} // namespace + +// Differential fuzz test: for random regexes and random rows, compare RE2's ground truth with the +// result of "evaluating the compiled GramQuery over that row's gram set". The one forbidden +// combination is truth=true with cand=false (the compiler may only give false positives, never +// false negatives: truth => cand); the reverse, cand=true with truth=false, is entirely allowed +// and simply means the candidate row must be re-verified by the regexp expression above it. +// The four schemes DENSE/SPARSE x lower_case=false/true share one rng sequence, so all four see +// exactly the same distribution of (regex, row) pairs and the only variable is the scheme; 3000 +// regexes x 20 rows = 60k RE2 matches per scheme, 240k across the four. +TEST(RegexGramFuzzTest, CompiledQueryIsSuperset) { + std::mt19937 rng(20260903); + for (GramMode mode : {GramMode::DENSE, GramMode::SPARSE}) { + for (bool lc : {false, true}) { + GramScheme s; + s.mode = mode; + s.lower_case = lc; + GramExtractor ex(s); + RegexGramCompiler comp(s); + int compiled = 0, indexable = 0; + for (int it = 0; it < 3000; it++) { + std::string re = random_regex(rng, 2); + check_regex(mode, lc, ex, comp, rng, re, &compiled, &indexable); + if (::testing::Test::HasFatalFailure()) { + return; + } + } + // Non-fatal coverage assertions guarding against a degenerate generator/compiler: at + // least 1000 of the 3000 random regexes must pass RE2's syntax check (the generator + // is not producing mostly invalid regexes), and at least 1/4 of the compilations must + // not be ALL (the compiler has not given up filtering on nearly every regex). + EXPECT_GT(compiled, 1000); + EXPECT_GT(indexable, compiled / 4) << "compiler is too conservative"; + } + } +} + +} // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/regex_gram_no_false_negative_test.cpp b/be/test/storage/index/inverted/gram/regex_gram_no_false_negative_test.cpp new file mode 100644 index 00000000000000..a433a1ba8e4bb0 --- /dev/null +++ b/be/test/storage/index/inverted/gram/regex_gram_no_false_negative_test.cpp @@ -0,0 +1,990 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Property test for the one hard contract of the gram compiler: +// +// for every row R and every pattern P, if the real engine says "P matches R" then +// evaluating compile(P, scheme) over extract(R, scheme) must yield true. +// +// A false positive (a candidate row the expression later rejects) is fine; a false negative +// silently drops a row from the result set, which is a correctness bug. The existing +// differential fuzz test (regex_gram_fuzz_test.cpp) only ever draws rows from a handful of +// pure-ASCII / valid-UTF-8 templates, so whole families of inputs -- illegal UTF-8 bytes, +// literals that start in the middle of a code point, NUL bytes, rows around the max_len +// boundary, very long rows -- were never exercised. This test walks a fixed corpus that +// deliberately contains all of them, across a scheme matrix that also varies min_len, max_len +// and density instead of only mode x lower_case. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/inverted/gram/gram_extractor.h" +#include "storage/index/inverted/gram/regex_gram_compiler.h" + +namespace doris::segment_v2::gram { + +namespace { + +// --------------------------------------------------------------------------------------------- +// Independent interpreters (they never call into the code under test) +// --------------------------------------------------------------------------------------------- + +// Evaluate a query tree over the gram set of one row, re-implemented from the Op semantics so +// that this test does not verify the compiler with the compiler's own simplification logic. +bool eval_query(const GramQuery& q, const std::set& grams) { + switch (q.op) { + case GramQuery::Op::ALL: + return true; + case GramQuery::Op::NONE: + return false; + case GramQuery::Op::AND: + return std::ranges::all_of(q.grams, [&](const auto& g) { return grams.contains(g); }) && + std::ranges::all_of(q.subs, [&](const auto& s) { return eval_query(s, grams); }); + case GramQuery::Op::OR: + return std::ranges::any_of(q.grams, [&](const auto& g) { return grams.contains(g); }) || + std::ranges::any_of(q.subs, [&](const auto& s) { return eval_query(s, grams); }); + } + return true; +} + +// Doris LIKE is byte oriented and anchored (the whole value has to match). `%` stands for any +// byte sequence, `_` for exactly one byte, and only `\%`, `\_` and `\\` are escapes. Modelling +// `_` as one byte rather than one code point makes the ground truth strictly more permissive, +// which can only make the "truth => candidate" assertion harder to satisfy -- and it stays sound, +// because the compiler treats `_` purely as a segment separator and assumes nothing about what it +// matches. +enum class LikeTok : uint8_t { kByte, kAnyOne, kAnySeq }; + +struct LikeItem { + LikeTok kind = LikeTok::kByte; + char byte = 0; +}; + +// `\x` where x is none of % _ \ has two possible readings (the engine may keep or drop the +// backslash); keep_unknown_backslash selects one of them. The caller checks both, because the +// compiler is conservative enough to have to be correct under either. +std::vector parse_like(std::string_view pat, bool keep_unknown_backslash) { + std::vector out; + for (size_t i = 0; i < pat.size(); i++) { + const char c = pat[i]; + if (c == '\\' && i + 1 < pat.size()) { + const char n = pat[i + 1]; + if (n == '%' || n == '_' || n == '\\') { + out.push_back({.kind = LikeTok::kByte, .byte = n}); + i++; + continue; + } + if (keep_unknown_backslash) { + out.push_back({.kind = LikeTok::kByte, .byte = '\\'}); + } + continue; // n is consumed as an ordinary character by the next iteration + } + if (c == '%') { + out.push_back({.kind = LikeTok::kAnySeq, .byte = 0}); + continue; + } + if (c == '_') { + out.push_back({.kind = LikeTok::kAnyOne, .byte = 0}); + continue; + } + out.push_back({.kind = LikeTok::kByte, .byte = c}); + } + return out; +} + +// Classic O(n*m) wildcard match, anchored at both ends. +bool like_match_tokens(const std::vector& t, std::string_view s) { + const size_t n = t.size(); + const size_t m = s.size(); + std::vector prev(m + 1, 0); + std::vector cur(m + 1, 0); + prev[0] = 1; + for (size_t i = 1; i <= n; i++) { + const LikeItem& it = t[i - 1]; + if (it.kind == LikeTok::kAnySeq) { + cur[0] = prev[0]; + for (size_t j = 1; j <= m; j++) { + cur[j] = static_cast(prev[j] != 0 || cur[j - 1] != 0); + } + } else { + cur[0] = 0; + for (size_t j = 1; j <= m; j++) { + const bool hit = + it.kind == LikeTok::kAnyOne || + static_cast(it.byte) == static_cast(s[j - 1]); + cur[j] = static_cast(hit && prev[j - 1] != 0); + } + } + prev.swap(cur); + } + return prev[m] != 0; +} + +bool like_matches(std::string_view pattern, std::string_view row) { + return like_match_tokens(parse_like(pattern, false), row) || + like_match_tokens(parse_like(pattern, true), row); +} + +// --------------------------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------------------------- + +std::string hex_escape(std::string_view s) { + static const char* kHex = "0123456789ABCDEF"; + std::string r; + for (unsigned char c : s) { + if (c >= 0x20 && c < 0x7F && c != '\\') { + r.push_back(static_cast(c)); + } else { + r += "\\x"; + r.push_back(kHex[c >> 4]); + r.push_back(kHex[c & 0x0F]); + } + } + return r; +} + +// A byte sequence no conforming UTF-8 encoder can ever produce, so no index can ever hold a gram +// containing it: a lead byte above 0xF4, a 0xF4 sequence encoding more than U+10FFFF, or the +// WTF-8 encoding of a surrogate. Requiring such a gram prunes every row unconditionally. +bool is_unencodable(std::string_view g) { + for (size_t i = 0; i < g.size(); i++) { + const auto c = static_cast(g[i]); + if (c >= 0xF5) { + return true; + } + if (c == 0xF4 && i + 1 < g.size() && static_cast(g[i + 1]) >= 0x90) { + return true; + } + if (c == 0xED && i + 1 < g.size() && static_cast(g[i + 1]) >= 0xA0) { + return true; + } + } + return false; +} + +bool query_has_unencodable_gram(const GramQuery& q) { + return std::ranges::any_of(q.grams, [](const auto& g) { return is_unencodable(g); }) || + std::ranges::any_of(q.subs, [](const auto& s) { return query_has_unencodable_gram(s); }); +} + +std::set grams_of_row(GramExtractor& ex, const std::string& row) { + std::vector g; + ex.extract(row, &g); + return {g.begin(), g.end()}; +} + +std::string describe(const GramScheme& s) { + return std::string(s.mode == GramMode::DENSE ? "dense" : "sparse") + "/lc" + + std::to_string(static_cast(s.lower_case)) + "/min" + std::to_string(s.min_len) + + "/max" + std::to_string(s.max_len) + "/d" + std::to_string(s.density_permille); +} + +// --------------------------------------------------------------------------------------------- +// Corpus +// --------------------------------------------------------------------------------------------- + +// mode x lower_case x min_len x max_len x density. DENSE ignores max_len/density, so those +// combinations are collapsed. min_len=1 is what drives RegexGramCompiler's `keep == 0` branch, +// which no other test reaches. +std::vector build_schemes() { + std::vector out; + for (GramMode mode : {GramMode::DENSE, GramMode::SPARSE}) { + for (bool lc : {false, true}) { + for (uint32_t min_len : {1U, 3U, 4U}) { + if (mode == GramMode::DENSE) { + GramScheme s; + s.mode = mode; + s.lower_case = lc; + s.min_len = min_len; + out.push_back(s); + continue; + } + for (uint32_t max_len : {4U, 16U}) { + for (uint32_t density : {50U, 250U, 1000U}) { + GramScheme s; + s.mode = mode; + s.lower_case = lc; + s.min_len = min_len; + s.max_len = max_len; + s.density_permille = density; + out.push_back(s); + } + } + } + } + } + return out; +} + +// Rows deliberately covering what the existing fuzz corpus never produced: illegal lead bytes, +// stray continuation bytes, truncated sequences, surrogate and overlong encodings, embedded NUL, +// lengths right at min_len/max_len, and rows far longer than any gram window. +std::vector build_rows() { + return { + "", + "a", + "ab", + "abc", + "abcd", + "abcde", + "abcdefghijklmno", // max_len - 1 + "abcdefghijklmnop", // max_len + "abcdefghijklmnopq", // max_len + 1 + "AbCdEf", + "ABC", + "abc abc abc", + "aaaaaaa", + "a\tb\nc", + " ", + "100%", + "a_b", + "a\\b", + "back\\slash", + "rpc error: code = Unavailable desc = timeout", + "user_id=abc GET /images/x.gif", + "code=Unavailable", + "timeout after error error error", + "GET /a/b/c?d=1&e=2", + "\xE6\x89\x8B\xE6\x9C\xBA\xE5\xBE\xAE\xE5\x8D\x9A POST 10.68.3.18:8080 error", + "\xC3\xA9" + "abc", // e-acute followed by "abc": the P0-2 row + "x\xC3\xA9" + "abcy", + "\xC3\xA9" + "lan vital", + "\xF0\x9F\x98\x80 emoji", + "\xFF", + "\xFF" + "abc", + "\xC3", + "z\xC3z", + "\xC3" + "abc", + "z\xC3" + "abcz", + "\xE4\xBD", + "z\xE4\xBDz", + "\xE4\xBD" + "abc", + "z\xE4\xBD" + "abcz", + "abc\xFF" + "def", + "xxabc\xFF" + "defyy", + "\xA9" + "abc", + "abc\xA9" + "def", + "abc\xC3", + "abc\xE4\xBD", + "\xED\xA0\x80" + "abc", // surrogate (WTF-8) encoding + "\xC0\xAF", // overlong '/' + std::string("ab\0cd", 5), + std::string("\0abc", 4), + std::string("abc\0", 4), + std::string(1, '\0'), + std::string("\xFF\0\xFE", 3), + std::string(4096, 'a'), + std::string(2000, 'x') + + "abc\xFF" + "def" + + std::string(2000, 'y'), + std::string(2000, 'x') + + "\xC3\xA9" + "abc" + + std::string(2000, 'y'), + }; +} + +// Regex patterns. The last block holds patterns carrying raw illegal bytes; they are written so +// that a byte-oriented and a code-point-oriented reading agree (no quantifier, class or `.` ever +// applies to a multi-byte run), which is what makes the Latin-1 ground truth used for them below +// legitimate. +std::vector build_regex_patterns() { + return { + "abc", + "abcdef", + "error", + "GET|POST", + "abc.*def", + "(abc|def)", + "^abc", + "abc$", + "a{3}b", + "(abc){2}", + "abc+", + "[abc]def", + "[^a]bcd", + "\\d+error", + "error.*timeout", + "user_id=[a-z]+", + "10\\.68\\.", + "(?i)ABC", + "(?i)\xE6\x89\x8B\xE6\x9C\xBA", + "\xE6\x89\x8B\xE6\x9C\xBA", + "\xE5\xBE\xAE\xE5\x8D\x9A", + "\xF0\x9F\x98\x80", + "\xC3\xA9" + "lan", + "aaaaaaaaaaaaaaaaaaaaaa", + "a", + "ab", + // Demotion paths: a Cartesian product over kMaxSet, an exact set over kMaxExact, and + // a long alternation. None of these had any coverage before. + "(a|b|c|d|e|f|g|h)(1|2|3)", + "(aa|bb|cc|dd|ee|ff|gg|hh)", + "(aaa|bbb|ccc|ddd|eee|fff|ggg|hhh|iii|jjj|kkk|lll|mmm|nnn|ooo|ppp|qqq|rrr|sss|ttt|" + "uuu|vvv|www|xxx|yyy)", + "(x|y)(1|2)(3|4)(5|6)(7|8)", + "(abc|abcd|abcde|abcdef|abcdefg|abcdefgh)", + // Raw illegal bytes in the pattern. + "abc\xFF" + "def", + "\xFF" + "abc", + "abc\xC3", + "a\xA9" + "b", + "\xC3" + "abc", + "\xE4\xBD" + "abc", + "\xC3" + "|abc", + "\xE4\xBD" + "|abc", + "\xFF" + "|abc", + }; +} + +std::vector build_like_patterns() { + return { + "abc", + "%abc%", + "%abc", + "abc%", + "a%c", + "%abc%def%", + "_abc_", + "%a_c%", + "abcdefghijklmnop", + "%abcdefghijklmnop%", + "%error%timeout%", + "%GET%", + "%code=Unavailable%", + "%\xE6\x89\x8B\xE6\x9C\xBA%", + "%\xE5\xBE\xAE\xE5\x8D\x9A%", + "%\xC3\xA9" + "abc%", + "%100\\%%", + "%a\\_b%", + "%a\\\\b%", + "", + "%", + "%%", + "%aaaaaaa%", + // Literal segments that are not aligned to a code point boundary, or that carry + // illegal bytes. + "%\xA9" + "abc%", + "%abc\xFF" + "def%", + "%\xC3%", + "%\xFF%", + "%\xED\xA0\x80%", + "%\xC0\xAF%", + "%\xE4\xBD%", + "%abc\xC3%", + "%\xC3" + "abc%", + "%\xE4\xBD" + "abc%", + }; +} + +// RE2 in UTF-8 mode is the ground truth for REGEXP (Doris compiles with HS_FLAG_UTF8 and falls +// back to RE2). A pattern holding raw illegal bytes cannot be compiled in UTF-8 mode at all, so +// for those the byte-oriented Latin-1 mode is used instead -- see the note on +// build_regex_patterns for why that stays faithful for this corpus. +std::unique_ptr build_ground_truth_regex(const std::string& pattern) { + auto utf8 = std::make_unique(pattern, RE2::Quiet); + if (utf8->ok()) { + return utf8; + } + RE2::Options opt; + opt.set_log_errors(false); + opt.set_encoding(RE2::Options::EncodingLatin1); + auto latin1 = std::make_unique(pattern, opt); + if (latin1->ok()) { + return latin1; + } + return nullptr; +} + +constexpr size_t kMaxRecordedFailures = 200; +constexpr size_t kReportedFailures = 25; + +void record(std::vector* failures, const std::string& msg) { + if (failures->size() < kMaxRecordedFailures) { + failures->push_back(msg); + } +} + +std::string join_failures(const std::vector& failures) { + std::string s = "false negatives: " + std::to_string(failures.size()) + + (failures.size() >= kMaxRecordedFailures ? "+ (capped)" : "") + "\n"; + for (size_t i = 0; i < failures.size() && i < kReportedFailures; i++) { + s += " " + failures[i] + "\n"; + } + return s; +} + +// The corpus plus its per-(pattern, row) ground truth, which does not depend on the scheme and is +// therefore computed once. +struct Corpus { + std::vector rows; + std::vector regex_patterns; + std::vector like_patterns; + std::vector> res; // null when neither RE2 mode can compile the pattern + std::vector> regex_truth; + std::vector> like_truth; + size_t compilable = 0; +}; + +Corpus build_corpus() { + Corpus c; + c.rows = build_rows(); + c.regex_patterns = build_regex_patterns(); + c.like_patterns = build_like_patterns(); + + c.res.resize(c.regex_patterns.size()); + c.regex_truth.resize(c.regex_patterns.size()); + for (size_t p = 0; p < c.regex_patterns.size(); p++) { + c.res[p] = build_ground_truth_regex(c.regex_patterns[p]); + if (c.res[p] == nullptr) { + continue; + } + c.compilable++; + c.regex_truth[p].resize(c.rows.size()); + for (size_t r = 0; r < c.rows.size(); r++) { + c.regex_truth[p][r] = static_cast(RE2::PartialMatch(c.rows[r], *c.res[p])); + } + } + + c.like_truth.resize(c.like_patterns.size()); + for (size_t p = 0; p < c.like_patterns.size(); p++) { + c.like_truth[p].resize(c.rows.size()); + for (size_t r = 0; r < c.rows.size(); r++) { + c.like_truth[p][r] = static_cast(like_matches(c.like_patterns[p], c.rows[r])); + } + } + return c; +} + +using GramSets = std::vector>; + +void check_one_pattern(const Corpus& c, const GramScheme& s, const GramSets& row_grams, + const std::string& kind, const std::string& pattern, + const std::vector& truth, const GramQuery& q, + std::vector* failures) { + for (size_t r = 0; r < c.rows.size(); r++) { + if (truth[r] == 0 || eval_query(q, row_grams[r])) { + continue; + } + record(failures, describe(s) + " " + kind + " '" + hex_escape(pattern) + "' row='" + + hex_escape(c.rows[r]) + "' q=" + hex_escape(q.to_debug_string())); + } +} + +void check_scheme(const Corpus& c, const GramScheme& s, std::vector* failures) { + GramExtractor ex(s); + RegexGramCompiler comp(s); + GramSets row_grams(c.rows.size()); + for (size_t r = 0; r < c.rows.size(); r++) { + row_grams[r] = grams_of_row(ex, c.rows[r]); + } + for (size_t p = 0; p < c.regex_patterns.size(); p++) { + if (c.res[p] == nullptr) { + continue; + } + GramQuery q; + EXPECT_TRUE(comp.compile_regexp(c.regex_patterns[p], &q).ok()); + check_one_pattern(c, s, row_grams, "REGEXP", c.regex_patterns[p], c.regex_truth[p], q, + failures); + } + for (size_t p = 0; p < c.like_patterns.size(); p++) { + GramQuery q; + EXPECT_TRUE(comp.compile_like(c.like_patterns[p], &q).ok()); + check_one_pattern(c, s, row_grams, "LIKE", c.like_patterns[p], c.like_truth[p], q, + failures); + } +} + +} // namespace + +// The core property: over the whole scheme matrix, no compiled query may ever be false on a row +// the engine actually matches. +TEST(RegexGramNoFalseNegativeTest, CompiledQueryNeverPrunesAMatchingRow) { + const Corpus c = build_corpus(); + // Guard against the corpus silently degenerating into "nothing has a ground truth". + ASSERT_GT(c.compilable, c.regex_patterns.size() * 3 / 4); + + std::vector failures; + for (const GramScheme& s : build_schemes()) { + check_scheme(c, s, &failures); + } + EXPECT_TRUE(failures.empty()) << join_failures(failures); +} + +// P0-1: a raw illegal byte in a regex is decoded into the fake code point 0x110000+byte and then +// re-encoded by encode_cp into a four-byte sequence above U+10FFFF. The write side slices raw +// bytes and never re-encodes anything, so that gram cannot exist in any index and the AND prunes +// every row of the query. +TEST(RegexGramNoFalseNegativeTest, IllegalUtf8ByteInRegexIsNotCompiledIntoAnImpossibleGram) { + GramScheme s; + s.mode = GramMode::DENSE; + s.min_len = 3; + GramExtractor ex(s); + RegexGramCompiler comp(s); + + GramQuery q; + ASSERT_TRUE(comp.compile_regexp("abc\xFF" + "def", + &q) + .ok()); + EXPECT_FALSE(query_has_unencodable_gram(q)) << hex_escape(q.to_debug_string()); + + const std::string row = + "xxabc\xFF" + "defyy"; + EXPECT_TRUE(eval_query(q, grams_of_row(ex, row))) + << "row '" << hex_escape(row) << "' contains the pattern byte for byte but the " + << "compiled query prunes it: " << hex_escape(q.to_debug_string()); +} + +// P0-2: GramExtractor splits position-dependently -- a byte run starting in the middle of a code +// point is split differently from the same bytes inside the whole row. A LIKE literal segment +// that is not aligned to a code point boundary therefore demands grams the row does not have, +// even though Doris LIKE is byte oriented and does match the row. +TEST(RegexGramNoFalseNegativeTest, LikeLiteralStartingInsideACodePointDoesNotPruneAMatchingRow) { + const std::string row = + "\xC3\xA9" + "abc"; // 'e-acute' + "abc" + const std::string pattern = + "%\xA9" + "abc%"; + ASSERT_TRUE(like_matches(pattern, row)); + + for (GramMode mode : {GramMode::DENSE, GramMode::SPARSE}) { + GramScheme s; + s.mode = mode; + s.min_len = 3; + GramExtractor ex(s); + RegexGramCompiler comp(s); + GramQuery q; + ASSERT_TRUE(comp.compile_like(pattern, &q).ok()); + EXPECT_TRUE(eval_query(q, grams_of_row(ex, row))) + << describe(s) << " LIKE '" << hex_escape(pattern) << "' prunes matching row '" + << hex_escape(row) << "'; q=" << hex_escape(q.to_debug_string()); + } +} + +// P0-3: Parser::next_cp advances by the length guessed from the lead byte, while decode_cps +// consumes exactly one byte for an ill-formed sequence. The bytes in between are swallowed -- +// regex metacharacters included -- so a different pattern gets compiled than the engine sees. +// `\xC3|abc` is "illegal byte OR abc"; swallowing the `|` turns it into the concatenation +// "illegal byte followed by abc" and forces the gram "abc" onto rows that only hold `\xC3`. +TEST(RegexGramNoFalseNegativeTest, TruncatedUtf8SequenceDoesNotSwallowTheNextPatternByte) { + GramScheme s; + s.mode = GramMode::DENSE; + s.min_len = 3; + GramExtractor ex(s); + RegexGramCompiler comp(s); + + struct Case { + std::string pattern; + std::string row; + }; + const std::vector cases = { + {.pattern = "\xC3" + "|abc", + .row = "z\xC3z"}, + {.pattern = "\xE4\xBD" + "|abc", + .row = "z\xE4\xBDz"}, + {.pattern = "\xC3" + "abc", + .row = "z\xC3" + "abcz"}, + {.pattern = "\xE4\xBD" + "abc", + .row = "z\xE4\xBD" + "abcz"}, + {.pattern = "\xFF" + "abc", + .row = "z\xFF" + "abcz"}, + }; + for (const Case& c : cases) { + GramQuery q; + ASSERT_TRUE(comp.compile_regexp(c.pattern, &q).ok()); + EXPECT_FALSE(query_has_unencodable_gram(q)) + << hex_escape(c.pattern) << " -> " << hex_escape(q.to_debug_string()); + EXPECT_TRUE(eval_query(q, grams_of_row(ex, c.row))) + << "REGEXP '" << hex_escape(c.pattern) << "' prunes matching row '" + << hex_escape(c.row) << "'; q=" << hex_escape(q.to_debug_string()); + } +} + +// An out-of-range `\x{...}` escape is plain ASCII in the pattern, yet it also produces a fake +// code point; the same degradation has to cover it. +TEST(RegexGramNoFalseNegativeTest, OutOfRangeHexEscapeDoesNotBecomeAnImpossibleGram) { + GramScheme s; + s.mode = GramMode::DENSE; + s.min_len = 3; + RegexGramCompiler comp(s); + for (const char* pattern : {"abc\\x{110000}def", "abc[\\x{110000}]def", "abc\\x{D800}def"}) { + GramQuery q; + ASSERT_TRUE(comp.compile_regexp(pattern, &q).ok()); + EXPECT_FALSE(query_has_unencodable_gram(q)) + << pattern << " -> " << hex_escape(q.to_debug_string()); + } +} + +// `(?<=` and `(?' inside the assertion body +// and leaves the rest of that body to be parsed as a consuming subexpression, turning a +// zero-width assertion into a mandatory literal. The row below really matches, so the compiled +// query must not prune it. +TEST(RegexGramNoFalseNegativeTest, LookbehindIsNotParsedAsANamedGroup) { + GramScheme s; + s.mode = GramMode::DENSE; + s.min_len = 3; + GramExtractor ex(s); + RegexGramCompiler comp(s); + + const std::string row = "aq>xy"; + for (const char* pattern : {"aq>x(?<=q>x)y", "aq>x(?w)y"}) { + GramQuery q; + ASSERT_TRUE(comp.compile_regexp(pattern, &q).ok()); + EXPECT_TRUE(eval_query(q, grams_of_row(ex, row))) + << "REGEXP '" << pattern << "' prunes matching row '" << row + << "'; q=" << hex_escape(q.to_debug_string()); + } + + // A real named group must keep working: it is a consuming subexpression, so its literal may + // still be folded into the query. + GramQuery named; + ASSERT_TRUE(comp.compile_regexp("(?abc)def", &named).ok()); + EXPECT_TRUE(eval_query(named, grams_of_row(ex, "xxabcdefyy"))); +} + +// Every leaf of the compiled tree costs a dictionary lookup and a posting read, which on object +// storage are separate remote requests. kMaxSet bounds one set but not the tree, so the compiler +// applies a whole-tree budget; no pattern may hand the query side more leaves than that. +TEST(RegexGramNoFalseNegativeTest, CompiledQueryStaysInsideTheGramBudget) { + GramScheme s; + s.mode = GramMode::DENSE; + s.min_len = 3; + GramExtractor ex(s); + RegexGramCompiler comp(s); + + // A wide alternation of distinct literals is the shape that multiplies bounded sets into an + // unbounded tree. + std::string wide = "("; + for (int i = 0; i < 40; i++) { + if (i > 0) { + wide += "|"; + } + wide += "lit" + std::to_string(i) + "xyz"; + } + wide += ")"; + + const std::vector patterns = {wide, "abcdef", "(ab|cd)(ef|gh)(ij|kl)", + "prefix.*suffix"}; + for (const auto& pattern : patterns) { + GramQuery q; + ASSERT_TRUE(comp.compile_regexp(pattern, &q).ok()) << pattern; + EXPECT_LE(q.leaf_count(), RegexGramCompiler::kMaxQueryGrams) << pattern; + } + + // Degrading over budget must stay on the safe side: a row the wide pattern really matches is + // still a candidate. + GramQuery q; + ASSERT_TRUE(comp.compile_regexp(wide, &q).ok()); + EXPECT_TRUE(eval_query(q, grams_of_row(ex, "zz lit7xyz zz"))); +} + +// --------------------------------------------------------------------------------------------- +// Demotion thresholds (kMaxSet / kMaxExact) +// --------------------------------------------------------------------------------------------- + +namespace { + +// The compiler carries finite enumerations (exact / prefix / suffix) and bounds each of them: a +// Cartesian product or union wider than kMaxSet, and an exact set larger than kMaxExact, are +// "demoted" down to prefix/suffix. Demotion is the classic place to lose rows -- a truncated set +// used as if it were complete turns "one of these" into "all of these" -- and neither threshold +// had a test sitting on it. +// +// Only sets whose elements are shorter than min_len can reach kMaxSet at all: a set whose every +// element already yields a gram is demoted at kMaxLongExact (4 elements), so the product of two +// such sets never passes sixteen. For the same reason kMaxExact is reachable only for a set that +// mixes gram-yielding and too-short elements. Hence the two-byte token pools below, always used +// with min_len == 3. +const std::vector& threshold_left_tokens() { + static const std::vector kTokens = {"aa", "bb", "cc", "dd", "ee", "ff", "gg"}; + return kTokens; +} + +const std::vector& threshold_right_tokens() { + static const std::vector kTokens = {"11", "22", "33", "44", "55"}; + return kTokens; +} + +// "a" is shorter than min_len, so this set is never "all long" and the kMaxLongExact rule cannot +// fire -- the only way for a set to survive up to kMaxExact. +const std::vector& threshold_mixed_tokens() { + static const std::vector kTokens = {"a", "bb", "cc", "dd", "ee", "ff", "gg", "hh"}; + return kTokens; +} + +// Distinct three-byte tokens: each one yields a gram on its own, which is what drives the +// prefix/suffix trimming loop once the union outgrows kMaxSet. +std::vector long_tokens(size_t count) { + std::vector out; + for (size_t i = 0; i < count; i++) { + std::string t(3, static_cast('a' + static_cast(i % 26))); + t[2] = static_cast('a' + static_cast(i / 26)); + out.push_back(t); + } + return out; +} + +std::string alternation_of(const std::vector& tokens, size_t count) { + std::string s = "("; + for (size_t i = 0; i < count; i++) { + if (i > 0) { + s += "|"; + } + s += tokens[i]; + } + return s + ")"; +} + +struct ThresholdCase { + std::string name; + std::string pattern; + std::vector matching_rows; // rows the pattern really matches + std::vector decoy_rows; // rows it does not match +}; + +// `(l0|l1|...)(r0|r1|...)`, plus one row per concatenation and one decoy per concatenation whose +// two halves are pushed apart (the pattern demands them adjacent). +ThresholdCase two_stage_case(std::string name, const std::vector& left, size_t nl, + const std::vector& right, size_t nr) { + ThresholdCase c; + c.name = std::move(name); + c.pattern = alternation_of(left, nl) + alternation_of(right, nr); + for (size_t i = 0; i < nl; i++) { + for (size_t j = 0; j < nr; j++) { + c.matching_rows.push_back("zz" + left[i] + right[j] + "zz"); + c.decoy_rows.push_back("zz" + left[i] + "__" + right[j] + "zz"); + } + } + c.decoy_rows.emplace_back("nothing of interest here"); + return c; +} + +// A bare alternation of `count` distinct three-byte tokens. +ThresholdCase wide_alternation_case(std::string name, size_t count) { + ThresholdCase c; + c.name = std::move(name); + const std::vector tokens = long_tokens(count); + c.pattern = alternation_of(tokens, count); + for (const auto& t : tokens) { + c.matching_rows.push_back("zz" + t + "zz"); + } + c.decoy_rows.emplace_back("nothing of interest here"); + c.decoy_rows.emplace_back("0123456789"); + return c; +} + +std::vector build_threshold_cases() { + std::vector out; + // kMaxSet == 20: a 4x5 product lands exactly on it, 5x5 and 7x5 pass it. + out.push_back(two_stage_case("cross_at_kMaxSet", threshold_left_tokens(), 4, + threshold_right_tokens(), 5)); + out.push_back(two_stage_case("cross_over_kMaxSet", threshold_left_tokens(), 5, + threshold_right_tokens(), 5)); + out.push_back(two_stage_case("cross_far_over_kMaxSet", threshold_left_tokens(), 7, + threshold_right_tokens(), 5)); + // kMaxExact == 7: a mixed-length alternation of 7 keeps its exact set, one of 8 is demoted. + out.push_back(two_stage_case("exact_at_kMaxExact", threshold_mixed_tokens(), 7, + threshold_right_tokens(), 2)); + out.push_back(two_stage_case("exact_over_kMaxExact", threshold_mixed_tokens(), 8, + threshold_right_tokens(), 2)); + // Union at and past kMaxSet: past it the prefix/suffix trimming loop shortens the kept bytes + // until the set fits. + out.push_back(wide_alternation_case("union_at_kMaxSet", 20)); + out.push_back(wide_alternation_case("union_over_kMaxSet", 25)); + return out; +} + +// The reference scheme for the behavioural assertions: DENSE with min_len 3 makes the gram set of +// a row exactly its three-byte windows, so what each pattern must demand can be worked out by +// hand. +GramScheme threshold_reference_scheme(uint32_t min_len = 3) { + GramScheme s; + s.mode = GramMode::DENSE; + s.min_len = min_len; + return s; +} + +// Rows of `rows` that the compiled pattern keeps as candidates. +std::vector threshold_candidates(const GramScheme& s, const std::string& pattern, + const std::vector& rows) { + GramExtractor ex(s); + RegexGramCompiler comp(s); + GramQuery q; + EXPECT_TRUE(comp.compile_regexp(pattern, &q).ok()) << pattern; + std::vector out(rows.size(), 0); + for (size_t r = 0; r < rows.size(); r++) { + out[r] = static_cast(eval_query(q, grams_of_row(ex, rows[r]))); + } + return out; +} + +} // namespace + +// (1) The hard contract on both sides of every threshold: a demoted set may cost pruning power, +// never a row. Run over the whole scheme matrix, because how much a set is trimmed depends on +// min_len/max_len and on whether the mode samples grams. +TEST(RegexGramNoFalseNegativeTest, DemotionAcrossSetSizeThresholdsNeverPrunesAMatchingRow) { + std::vector failures; + for (const ThresholdCase& c : build_threshold_cases()) { + // The decoys are only meaningful if they really do not match, so RE2 confirms both + // halves of the corpus before it is used. + RE2 re(c.pattern, RE2::Quiet); + ASSERT_TRUE(re.ok()) << c.name << " " << c.pattern; + for (const auto& row : c.matching_rows) { + ASSERT_TRUE(RE2::PartialMatch(row, re)) << c.name << " row=" << row; + } + for (const auto& row : c.decoy_rows) { + ASSERT_FALSE(RE2::PartialMatch(row, re)) << c.name << " decoy=" << row; + } + for (const GramScheme& s : build_schemes()) { + const auto keep = threshold_candidates(s, c.pattern, c.matching_rows); + for (size_t r = 0; r < c.matching_rows.size(); r++) { + if (keep[r] == 0) { + record(&failures, describe(s) + " " + c.name + " REGEXP '" + c.pattern + + "' prunes matching row '" + + hex_escape(c.matching_rows[r]) + "'"); + } + } + } + } + EXPECT_TRUE(failures.empty()) << join_failures(failures); +} + +// (2) The threshold value itself: a set of exactly kMaxSet entries must still be enumerated. If +// the comparison ever slipped to ">=", the 4x5 product below would be demoted one element early +// and the query would stop demanding the grams that span the junction of the two halves -- which +// is invisible to a "no false negative" test and only shows up as lost pruning. +TEST(RegexGramNoFalseNegativeTest, ASetOfExactlyKMaxSetIsStillEnumerated) { + const GramScheme s = threshold_reference_scheme(); + const ThresholdCase at = two_stage_case("cross_at_kMaxSet", threshold_left_tokens(), 4, + threshold_right_tokens(), 5); + ASSERT_EQ(at.matching_rows.size(), RegexGramCompiler::kMaxSet); + + RegexGramCompiler comp(s); + GramQuery q; + ASSERT_TRUE(comp.compile_regexp(at.pattern, &q).ok()); + ASSERT_FALSE(q.is_all()) << "the product of 4x5 == kMaxSet entries must not be demoted: " + << q.to_debug_string(); + EXPECT_LE(q.leaf_count(), RegexGramCompiler::kMaxQueryGrams); + + const auto kept = threshold_candidates(s, at.pattern, at.matching_rows); + for (size_t r = 0; r < at.matching_rows.size(); r++) { + EXPECT_EQ(kept[r], 1) << "matching row pruned: " << at.matching_rows[r]; + } + // Every decoy splits its two halves apart, so none of them holds a gram spanning the + // junction; enumerating the product is exactly what lets the query reject them. + const auto decoys = threshold_candidates(s, at.pattern, at.decoy_rows); + for (size_t r = 0; r < at.decoy_rows.size(); r++) { + EXPECT_EQ(decoys[r], 0) << "decoy not pruned: " << at.decoy_rows[r]; + } +} + +// (3) Crossing a threshold may only ever widen the candidate set. The pattern pairs below are +// nested by construction (the wider one accepts every string the narrower one accepts), so a row +// the narrow query keeps must also survive the wide one; the reverse -- a wider language pruning +// more rows -- would mean a demotion path had started to trust a truncated set. +TEST(RegexGramNoFalseNegativeTest, CrossingADemotionThresholdOnlyWidensTheCandidateSet) { + struct Pair { + std::string name; + ThresholdCase narrow; + ThresholdCase wide; + }; + const std::vector pairs = { + {.name = "kMaxSet", + .narrow = + two_stage_case("at", threshold_left_tokens(), 4, threshold_right_tokens(), 5), + .wide = two_stage_case("over", threshold_left_tokens(), 5, threshold_right_tokens(), + 5)}, + {.name = "kMaxExact", + .narrow = + two_stage_case("at", threshold_mixed_tokens(), 7, threshold_right_tokens(), 2), + .wide = two_stage_case("over", threshold_mixed_tokens(), 8, threshold_right_tokens(), + 2)}, + {.name = "union", + .narrow = wide_alternation_case("at", 20), + .wide = wide_alternation_case("over", 25)}, + }; + // Restricted to DENSE, whose gram set is simply "every window of min_len bytes": the claim is + // about the compiler's demotion order, and a sampling mode would fold the extractor's own + // boundary choices into the comparison. + for (uint32_t min_len : {3U, 4U}) { + const GramScheme s = threshold_reference_scheme(min_len); + for (const Pair& p : pairs) { + std::vector universe = p.wide.matching_rows; + universe.insert(universe.end(), p.wide.decoy_rows.begin(), p.wide.decoy_rows.end()); + const auto narrow = threshold_candidates(s, p.narrow.pattern, universe); + const auto wide = threshold_candidates(s, p.wide.pattern, universe); + for (size_t r = 0; r < universe.size(); r++) { + EXPECT_FALSE(narrow[r] == 1 && wide[r] == 0) + << p.name << " min_len=" << min_len << ": row '" << hex_escape(universe[r]) + << "' survives '" << p.narrow.pattern << "' but is pruned by the wider '" + << p.wide.pattern << "'"; + } + } + } +} + +} // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp b/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp new file mode 100644 index 00000000000000..574b79ac3f8ff2 --- /dev/null +++ b/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp @@ -0,0 +1,846 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include +#include + +#include "core/block/block.h" +#include "core/column/column_const.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "exprs/function/like.h" +#include "exprs/function_context.h" +#include "exprs/vslot_ref.h" +#include "runtime/runtime_state.h" +#include "storage/index/inverted/gram/gram_extractor.h" +#include "storage/index/inverted/gram/regex_gram_compiler.h" + +namespace doris::segment_v2::gram { +namespace { + +enum class ScalarPath { + ANY, + HYPERSCAN, + RE2, + DYNAMIC_RE2, + BOOST, + ALLPASS, + EQUALS, + STARTS_WITH, + ENDS_WITH, + SUBSTRING +}; + +// BE UT permits private/protected access. Inspect the state selected by open(), without +// replacing the production engine or using a second regex implementation as the oracle. +void check_scalar_path(const LikeState& state, ScalarPath path) { + if (path == ScalarPath::ANY) { + return; + } + const auto& search = state.search_state; + EXPECT_EQ(search.hs_database != nullptr, path == ScalarPath::HYPERSCAN); + EXPECT_EQ(search.hs_scratch != nullptr, path == ScalarPath::HYPERSCAN); + EXPECT_EQ(search.regex != nullptr, path == ScalarPath::RE2); + EXPECT_EQ(search.boost_regex != nullptr, path == ScalarPath::BOOST); + + using FunctionPointer = Status (*)(const LikeSearchState*, const ColumnString&, + const StringRef&, ColumnUInt8::Container&); + const auto* selected = state.function.target(); + ASSERT_NE(selected, nullptr); + switch (path) { + case ScalarPath::HYPERSCAN: + case ScalarPath::RE2: + case ScalarPath::BOOST: + EXPECT_EQ(*selected, &FunctionLikeBase::constant_regex_fn); + break; + case ScalarPath::DYNAMIC_RE2: { + using ScalarFunctionPointer = Status (*)(const LikeSearchState*, const StringRef&, + const StringRef&, unsigned char*); + const auto* scalar = state.scalar_function.target(); + ASSERT_NE(scalar, nullptr); + EXPECT_EQ(*selected, + state.is_like_pattern ? &FunctionLike::like_fn : &FunctionLikeBase::regexp_fn); + EXPECT_EQ(*scalar, state.is_like_pattern ? &FunctionLike::like_fn_scalar + : &FunctionLikeBase::regexp_fn_scalar); + break; + } + case ScalarPath::ALLPASS: + EXPECT_EQ(*selected, &FunctionLikeBase::constant_allpass_fn); + break; + case ScalarPath::EQUALS: + EXPECT_EQ(*selected, &FunctionLikeBase::constant_equals_fn); + break; + case ScalarPath::STARTS_WITH: + EXPECT_EQ(*selected, &FunctionLikeBase::constant_starts_with_fn); + break; + case ScalarPath::ENDS_WITH: + EXPECT_EQ(*selected, &FunctionLikeBase::constant_ends_with_fn); + break; + case ScalarPath::SUBSTRING: + EXPECT_EQ(*selected, &FunctionLikeBase::constant_substring_fn); + break; + case ScalarPath::ANY: + break; + } +} + +// The expression chooses the real scalar engine, including Hyperscan and its RE2 fallback. +// Calling RE2 directly would miss differences between the compiler and the production path. +template +Status scalar_matches(const std::string& pattern, const std::vector& rows, + std::vector* matches, bool enable_hyperscan_fallback = true, + bool enable_extended_regex = false, ScalarPath path = ScalarPath::ANY) { + TQueryOptions query_options; + query_options.__set_enable_hyperscan_fallback(enable_hyperscan_fallback); + query_options.__set_enable_extended_regex(enable_extended_regex); + RuntimeState runtime_state(query_options, TQueryGlobals {}); + auto string_type = std::make_shared(); + auto context = FunctionContext::create_context( + &runtime_state, std::make_shared(), {string_type, string_type}); + + auto values = ColumnString::create(); + for (const auto& row : rows) { + values->insert_data(row.data(), row.size()); + } + auto patterns = ColumnString::create(); + const bool dynamic_pattern = path == ScalarPath::DYNAMIC_RE2; + const size_t pattern_rows = dynamic_pattern ? rows.size() : 1; + for (size_t i = 0; i < pattern_rows; ++i) { + patterns->insert_data(pattern.data(), pattern.size()); + } + ColumnPtr pattern_column; + std::vector> constant_columns(2); + if (dynamic_pattern) { + // Prove vector_non_const will reach the scalar regex function, where RE2 is + // constructed locally. No scalar or vector string fast path accepts these patterns. + constexpr bool is_like = std::is_same_v; + EXPECT_EQ(FunctionLikeBase::pattern_type_recognition(*patterns), nullptr); + if constexpr (is_like) { + std::string literal; + EXPECT_EQ(extract_like_fast_path(pattern.data(), pattern.size(), literal), + LikeFastPath::REGEX); + } + pattern_column = std::move(patterns); + } else { + pattern_column = ColumnConst::create(std::move(patterns), rows.size()); + constant_columns[1] = std::make_shared(pattern_column); + } + context->set_constant_cols(constant_columns); + + Function function; + if (dynamic_pattern) { + // Dynamic-pattern execution is a scalar compatibility boundary, not an index + // acceleration claim: the eligibility hook rejects a pattern column. + auto value = VSlotRef::create_shared(0, 0, 0, string_type, "value"); + auto pattern_slot = VSlotRef::create_shared(1, 1, 1, string_type, "pattern"); + EXPECT_FALSE(function.can_evaluate_inverted_index({value, pattern_slot})); + } + RETURN_IF_ERROR(function.open(context.get(), FunctionContext::THREAD_LOCAL)); + const auto* state = + static_cast(context->get_function_state(FunctionContext::THREAD_LOCAL)); + check_scalar_path(*state, path); + Block block; + block.insert({std::move(values), string_type, "value"}); + block.insert({std::move(pattern_column), string_type, "pattern"}); + block.insert({nullptr, std::make_shared(), "result"}); + RETURN_IF_ERROR(function.execute_impl(context.get(), block, {0, 1}, 2, rows.size())); + matches->clear(); + for (size_t i = 0; i < rows.size(); ++i) { + matches->push_back(block.get_by_position(2).column->get_bool(i)); + unsigned char scalar_result = 0; + RETURN_IF_ERROR(state->scalar_function(&state->search_state, StringRef(rows[i]), + StringRef(pattern), &scalar_result)); + EXPECT_EQ(scalar_result != 0, matches->back()) << "row " << i; + } + return function.close(context.get(), FunctionContext::THREAD_LOCAL); +} + +// Interpret the public boolean query without reusing its compiler or simplifier. +bool is_candidate(const GramQuery& query, const std::set& grams) { + switch (query.op) { + case GramQuery::Op::ALL: + return true; + case GramQuery::Op::NONE: + return false; + case GramQuery::Op::AND: + return std::ranges::all_of(query.grams, + [&](const auto& gram) { return grams.contains(gram); }) && + std::ranges::all_of(query.subs, + [&](const auto& sub) { return is_candidate(sub, grams); }); + case GramQuery::Op::OR: + return std::ranges::any_of(query.grams, + [&](const auto& gram) { return grams.contains(gram); }) || + std::ranges::any_of(query.subs, + [&](const auto& sub) { return is_candidate(sub, grams); }); + } + ADD_FAILURE() << "Unknown gram query operator"; + return false; +} + +template +void check_scheme_recall(const GramScheme& scheme, const std::string& pattern, + const std::vector& rows, const std::vector& truth, + bool require_pruning, bool require_all = false) { + SCOPED_TRACE(scheme.cache_key()); + RegexGramCompiler compiler(scheme); + GramExtractor extractor(scheme); + GramQuery query; + if constexpr (std::is_same_v) { + ASSERT_TRUE(compiler.compile_like(pattern, &query).ok()); + } else { + ASSERT_TRUE(compiler.compile_regexp(pattern, &query).ok()); + } + if (require_all) { + EXPECT_EQ(query.op, GramQuery::Op::ALL) << query.to_debug_string(); + } + bool pruned = false; + for (size_t i = 0; i < rows.size(); ++i) { + SCOPED_TRACE(rows[i]); + std::vector extracted; + extractor.extract(rows[i], &extracted); + std::set grams(extracted.begin(), extracted.end()); + const bool candidate = is_candidate(query, grams); + EXPECT_TRUE(!truth[i] || candidate) << query.to_debug_string(); + pruned |= !truth[i] && !candidate; + } + if (require_pruning) { + EXPECT_TRUE(pruned) << "A determined literal must still filter unrelated rows"; + } +} + +template +void check_recall(const std::string& pattern, const std::vector& rows, + const std::vector& expected, bool require_pruning, + bool enable_hyperscan_fallback = true, bool enable_extended_regex = false, + ScalarPath path = ScalarPath::ANY, bool require_all = false) { + SCOPED_TRACE(pattern); + std::vector truth; + auto status = scalar_matches(pattern, rows, &truth, enable_hyperscan_fallback, + enable_extended_regex, path); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(truth, expected); + for (auto mode : {GramMode::DENSE, GramMode::SPARSE}) { + for (bool lower_case : {false, true}) { + GramScheme scheme; + scheme.mode = mode; + scheme.lower_case = lower_case; + check_scheme_recall(scheme, pattern, rows, truth, require_pruning, + require_all); + } + } +} + +struct RecallCase { + std::string pattern; + std::vector rows; + std::vector expected; + ScalarPath path; + bool require_pruning = true; + bool require_all = false; +}; + +template +void check_cases(const std::vector& cases, bool fallback, bool extended) { + SCOPED_TRACE(fallback); + SCOPED_TRACE(extended); + for (const auto& test_case : cases) { + check_recall(test_case.pattern, test_case.rows, test_case.expected, + test_case.require_pruning, fallback, extended, test_case.path, + test_case.require_all); + } +} + +} // namespace + +TEST(RegexGramRecallTest, HyperscanEngineOptionsMatrix) { + const std::vector cases = { + {"^prefix[0-9]+timeout$", + {"prefix123timeout", "prefix9timeout", "prefixxtimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::HYPERSCAN}, + {"prefix.timeout", + {"prefix\ntimeout", "prefix😀timeout", "prefixtimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::HYPERSCAN}, + {"(?-s)prefix.timeout", + {"prefix\ntimeout", "prefix😀timeout", "prefixtimeout", "unrelated"}, + {false, true, false, false}, + ScalarPath::HYPERSCAN}, + {"(?m)^timeout$", + {"before\ntimeout\nafter", "timeout", "xtimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::HYPERSCAN}, + {"(?i:ask)(?-i:timeout)", + {"ASKtimeout", "aſktimeout", "asKtimeout", "ASKTIMEOUT", "unrelated"}, + {true, true, true, false, false}, + ScalarPath::HYPERSCAN}, + {"(?i)(?:(?-i)abc)DEF(?-i:timeout)", + {"abcdeftimeout", "abcDEFtimeout", "ABCdeftimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::HYPERSCAN}, + {R"(ab\vcdtimeout)", + {"ab\vcdtimeout", "ab\ncdtimeout", "ab\rcdtimeout", "ab\fcdtimeout", "unrelated"}, + {true, true, true, true, false}, + ScalarPath::HYPERSCAN}, + {R"(\Qab😀\E{2}timeout)", + {"ab😀😀timeout", "ab😀timeout", "unrelated"}, + {true, false, false}, + ScalarPath::HYPERSCAN}, + {R"(foo\x00timeout)", + {std::string("foo") + '\0' + "timeout", "foo", "unrelated"}, + {true, false, false}, + ScalarPath::HYPERSCAN}, + // Hyperscan permits zero hex digits after \x and emits a NUL byte. + // The gram parser requires two digits and conservatively returns ALL. + {R"(timeout\xZZ)", + {std::string("timeout") + '\0' + "ZZ", "timeoutxZZ", R"(timeout\xZZ)", "unrelated"}, + {true, false, false, false}, + ScalarPath::HYPERSCAN, + false, + true}, + {std::string("foo[0-9]") + '\0' + "timeout", + {"foo1", "foo1timeout", "foo", "unrelated"}, + {true, true, false, false}, + ScalarPath::HYPERSCAN, + false, + true}, + {R"(timeout\Z)", + {"timeout", "timeoutx", "unrelated"}, + {true, false, false}, + ScalarPath::HYPERSCAN}, + {R"(ab\hcdtimeout)", + {"ab cdtimeout", "ab\tcdtimeout", "ab\ncdtimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::HYPERSCAN, + false, + true}}; + for (bool fallback : {false, true}) { + for (bool extended : {false, true}) { + check_cases(cases, fallback, extended); + } + } +} + +TEST(RegexGramRecallTest, Re2FallbackEngineMatrix) { + // Bounded repetition above 50 is rejected before hs_compile. The optional suffix + // deliberately selects RE2 without changing the corner case's expected matches. + const std::vector cases = { + {"^prefix.{0,51}timeout$", + {"prefixtimeout", "prefix" + std::string(51, 'x') + "timeout", + "prefix" + std::string(52, 'x') + "timeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::RE2}, + {"(?i:é)timeout.{0,51}$", + {"étimeout", "Étimeout", "ÉTIMEOUT", "unrelated"}, + {true, true, false, false}, + ScalarPath::RE2}, + {"(?i:ask)(?-i:timeout).{0,51}$", + {"ASKtimeout", "aſktimeout", "asKtimeout", "ASKTIMEOUT", "unrelated"}, + {true, true, true, false, false}, + ScalarPath::RE2}, + {"(?i)(?:(?-i)abc)DEF(?-i:timeout).{0,51}$", + {"abcdeftimeout", "abcDEFtimeout", "ABCdeftimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::RE2}, + {"(?-s)prefix.timeout.{0,51}$", + {"prefix\ntimeout", "prefix😀timeout", "prefixtimeout", "unrelated"}, + {false, true, false, false}, + ScalarPath::RE2}, + {"(?m)^timeout$.{0,51}", + {"before\ntimeout\nafter", "timeout", "xtimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::RE2}, + {"(?U)prefix.*timeout.{0,51}$", + {"prefixtimeout", "prefix\ntimeout", "prefixTIMEOUT", "unrelated"}, + {true, true, false, false}, + ScalarPath::RE2}, + // RE2's \v matches only VT; Hyperscan's \v also matches LF, CR and FF. + {R"(ab\vcdtimeout.{0,51}$)", + {"ab\vcdtimeout", "ab\ncdtimeout", "ab\rcdtimeout", "ab\fcdtimeout", "unrelated"}, + {true, false, false, false, false}, + ScalarPath::RE2}, + {R"(ab[\141]cdtimeout.{0,51}$)", + {"abacdtimeout", "ab1cdtimeout", "unrelated"}, + {true, false, false}, + ScalarPath::RE2}, + {R"(\Qab😀\E{2}timeout.{0,51}$)", + {"ab😀😀timeout", "ab😀timeout", "unrelated"}, + {true, false, false}, + ScalarPath::RE2}, + // This fallback is caused by rejected syntax rather than repetition cost. + {R"(ab+\Q\E?timeout)", + {"atimeout", "abtimeout", "abbbtimeout", "unrelated"}, + {true, true, true, false}, + ScalarPath::RE2}, + {R"(foo\x00timeout.{0,51}$)", + {std::string("foo") + '\0' + "timeout", "foo", "unrelated"}, + {true, false, false}, + ScalarPath::RE2}, + {std::string("foo[0-9].{0,51}") + '\0' + "timeout", + {std::string("foo1") + '\0' + "timeout", "foo1", "foo1timeout", "unrelated"}, + {true, false, false, false}, + ScalarPath::RE2, + false, + true}}; + for (bool extended : {false, true}) { + check_cases(cases, true, extended); + for (const auto& test_case : cases) { + SCOPED_TRACE(test_case.pattern); + std::vector matches; + auto status = + scalar_matches(test_case.pattern, test_case.rows, &matches, false, extended); + EXPECT_FALSE(status.ok()) << "Disabling fallback must surface Hyperscan rejection"; + } + } +} + +TEST(RegexGramRecallTest, BoostExtendedEngineAndRejectionMatrix) { + // Each expression is unsupported by both Hyperscan and RE2. Boost is available only + // when both fallback and extended regex are enabled. The gram parser must return ALL. + const std::vector cases = { + {"timeout(?=END)", + {"timeoutEND", "timeoutend", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"timeout(?!END)", + {"timeoutend", "timeoutEND", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"(?<=prefix)timeout", + {"prefixtimeout", "xtimeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"(? matches; + auto status = scalar_matches(test_case.pattern, test_case.rows, &matches, fallback, + extended); + EXPECT_FALSE(status.ok()); + if (fallback) { + EXPECT_NE(status.to_string().find("enable_extended_regex=true"), + std::string::npos); + } + } + } + } +} + +TEST(RegexGramRecallTest, RegexpAndLikeFastPathMatrix) { + const std::vector rows = {"timeout", "Timeout", "xtimeout", + "timeoutx", "xtimeoutx", "unrelated"}; + const std::string nul_literal = std::string("foo") + '\0' + "timeout"; + const std::vector regexp_cases = { + {".*", + {"", "\n", nul_literal, "unrelated"}, + {true, true, true, true}, + ScalarPath::ALLPASS, + false, + true}, + {"^timeout$", rows, {true, false, false, false, false, false}, ScalarPath::EQUALS}, + {"^timeout.*", rows, {true, false, false, true, false, false}, ScalarPath::STARTS_WITH}, + {".*timeout$", rows, {true, false, true, false, false, false}, ScalarPath::ENDS_WITH}, + {".*timeout.*", rows, {true, false, true, true, true, false}, ScalarPath::SUBSTRING}, + {nul_literal, + {nul_literal, "foo", "unrelated"}, + {true, false, false}, + ScalarPath::SUBSTRING, + false, + true}}; + const std::vector like_cases = { + {"%%", + {"", "\n", nul_literal, "unrelated"}, + {true, true, true, true}, + ScalarPath::ALLPASS, + false, + true}, + {"timeout", rows, {true, false, false, false, false, false}, ScalarPath::EQUALS}, + {"timeout%", rows, {true, false, false, true, false, false}, ScalarPath::STARTS_WITH}, + {"%timeout", rows, {true, false, true, false, false, false}, ScalarPath::ENDS_WITH}, + {"%timeout%", rows, {true, false, true, true, true, false}, ScalarPath::SUBSTRING}, + {R"(prefix\_timeout)", + {"prefix_timeout", "prefixXtimeout", "unrelated"}, + {true, false, false}, + ScalarPath::EQUALS}, + {R"(%prefix\_timeout%)", + {"xprefix_timeoutx", "prefixXtimeout", "unrelated"}, + {true, false, false}, + ScalarPath::SUBSTRING}, + {R"(prefix\\timeout)", + {R"(prefix\timeout)", "prefixtimeout", "unrelated"}, + {true, false, false}, + ScalarPath::EQUALS}, + {nul_literal, + {nul_literal, "foo", "unrelated"}, + {true, false, false}, + ScalarPath::EQUALS, + false, + true}}; + for (bool fallback : {false, true}) { + for (bool extended : {false, true}) { + check_cases(regexp_cases, fallback, extended); + check_cases(like_cases, fallback, extended); + } + } +} + +TEST(RegexGramRecallTest, LikeHyperscanEngineMatrix) { + const std::vector cases = { + {"prefix_timeout", + {"prefix😀timeout", "prefix\ntimeout", "prefixtimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::HYPERSCAN}, + {"%prefix%timeout%", + {"prefix\ntimeout", "prefix😀timeout", "timeoutprefix", "unrelated"}, + {true, true, false, false}, + ScalarPath::HYPERSCAN}, + {R"(prefix\%timeout_)", + {"prefix%timeoutX", "prefixxtimeoutX", "unrelated"}, + {true, false, false}, + ScalarPath::HYPERSCAN}, + {R"(%prefix\qtimeout%)", + {R"(xprefix\qtimeoutx)", "prefixqtimeout", "unrelated"}, + {true, false, false}, + ScalarPath::HYPERSCAN}, + {"étimeout_", + {"étimeout😀", "Étimeout😀", "unrelated"}, + {true, false, false}, + ScalarPath::HYPERSCAN}, + {std::string("prefix_") + '\0' + "%timeout%", + {"prefix1", "prefix12", "xprefix1", "unrelated"}, + {true, true, false, false}, + ScalarPath::HYPERSCAN, + false, + true}}; + for (bool fallback : {false, true}) { + for (bool extended : {false, true}) { + check_cases(cases, fallback, extended); + } + } +} + +TEST(RegexGramRecallTest, InvalidPatternsRejectAcrossEngineOptions) { + // Unlike Hyperscan/RE2, Boost interprets \141 as backreference 1 followed by + // literal 41. This expression has no capture, so even extended regex rejects it. + for (const auto* pattern : {"[", "(abc", "(?i", "(?q)timeout", R"((?=ab)ab\141cdtimeout)"}) { + SCOPED_TRACE(pattern); + for (bool fallback : {false, true}) { + for (bool extended : {false, true}) { + SCOPED_TRACE(fallback); + SCOPED_TRACE(extended); + std::vector matches; + auto status = scalar_matches(pattern, {"timeout", "unrelated"}, &matches, fallback, + extended); + EXPECT_FALSE(status.ok()) << "Invalid syntax must not become a successful match"; + } + } + // Scalar rejection and conservative index compilation are separate contracts. + for (auto mode : {GramMode::DENSE, GramMode::SPARSE}) { + for (bool lower_case : {false, true}) { + GramScheme scheme; + scheme.mode = mode; + scheme.lower_case = lower_case; + check_scheme_recall(scheme, pattern, {"timeout", "unrelated"}, + {false, false}, false, true); + } + } + } +} + +TEST(RegexGramRecallTest, DynamicPatternRe2ExecutionAndIndexEligibility) { + // A ColumnString pattern reaches vector_non_const -> scalar_function -> RE2 even + // with Hyperscan fallback disabled. Extended regex does not enable Boost on this path. + // Each execution also checks that can_evaluate_inverted_index rejects the pattern slot. + const std::vector regexp_cases = { + {R"(ab\vcdtimeout)", + {"ab\vcdtimeout", "ab\ncdtimeout", "ab\rcdtimeout", "unrelated"}, + {true, false, false, false}, + ScalarPath::DYNAMIC_RE2}, + {"(?i:ask)(?-i:timeout)", + {"ASKtimeout", "aſktimeout", "asKtimeout", "ASKTIMEOUT", "unrelated"}, + {true, true, true, false, false}, + ScalarPath::DYNAMIC_RE2}, + {R"(\Qab😀\E{2}timeout)", + {"ab😀😀timeout", "ab😀timeout", "unrelated"}, + {true, false, false}, + ScalarPath::DYNAMIC_RE2}, + {std::string("foo[0-9]") + '\0' + "timeout", + {std::string("foo1") + '\0' + "timeout", "foo1", "foo1timeout", "unrelated"}, + {true, false, false, false}, + ScalarPath::DYNAMIC_RE2, + false, + true}}; + const std::vector like_cases = { + {"prefix_timeout", + {"prefix😀timeout", "prefix\ntimeout", "prefixtimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::DYNAMIC_RE2}, + {"%prefix%\ntimeout%", + {"prefix\ntimeout", "prefixx\ntimeout", "prefixxtimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::DYNAMIC_RE2}, + {R"(prefix\%timeout_)", + {"prefix%timeoutX", "prefixxtimeoutX", "unrelated"}, + {true, false, false}, + ScalarPath::DYNAMIC_RE2}, + {"étimeout_", + {"étimeout😀", "Étimeout😀", "unrelated"}, + {true, false, false}, + ScalarPath::DYNAMIC_RE2}, + {std::string("prefix_") + '\0' + "%timeout%", + {std::string("prefix1") + '\0' + "timeout", "prefix1", "prefix1timeout", "unrelated"}, + {true, false, false, false}, + ScalarPath::DYNAMIC_RE2, + false, + true}}; + for (bool fallback : {false, true}) { + for (bool extended : {false, true}) { + check_cases(regexp_cases, fallback, extended); + check_cases(like_cases, fallback, extended); + for (const auto* pattern : + {R"(timeout\Z)", R"(ab\hcdtimeout)", R"(timeout\xZZ)", "timeout(?=END)"}) { + SCOPED_TRACE(pattern); + std::vector matches; + auto status = scalar_matches(pattern, {"timeoutEND", "unrelated"}, &matches, + fallback, extended, ScalarPath::DYNAMIC_RE2); + EXPECT_FALSE(status.ok()) << "Dynamic patterns do not use Hyperscan or Boost"; + } + } + } +} + +TEST(RegexGramRecallTest, UnicodeCaseInsensitiveRecall) { + check_recall("(?i)é", {"é", "É", "unrelated"}, {true, true, false}, false); + check_recall("(?i)[é]", {"é", "É", "unrelated"}, {true, true, false}, false); + check_recall("(?i)ask", {"ask", "ASK", "aſk", "asK", "unrelated"}, + {true, true, true, true, false}, false); + check_recall("(?i)a[s]k", {"aſk", "asK", "unrelated"}, {true, true, false}, false); + check_recall("(?i)as[k]", {"aſk", "asK", "unrelated"}, {true, true, false}, false); + + // Unknown folded characters may weaken their own constraints, but must not disable the + // determined ASCII suffix. The short prefix also checks that grams do not cross that leaf. + check_recall("ab(?i)é(?-i:timeout)", {"abÉtimeout", "abétimeout", "unrelated"}, + {true, true, false}, true); + check_recall("ab(?i)[é](?-i:timeout)", {"abÉtimeout", "abétimeout", "unrelated"}, + {true, true, false}, true); + check_recall("timeout", {"timeout", "unrelated"}, {true, false}, true); +} + +TEST(RegexGramRecallTest, EscapesPreserveScalarRecall) { + check_recall(R"(ab\acdtimeout)", {"ab\acdtimeout", "abacdtimeout", "unrelated"}, + {true, false, false}, true); + check_recall(R"(ab\fcdtimeout)", {"ab\fcdtimeout", "abfcdtimeout", "unrelated"}, + {true, false, false}, true); + check_recall(R"(ab\vcdtimeout)", + {"ab\vcdtimeout", "ab\ncdtimeout", "ab\rcdtimeout", "ab\fcdtimeout", + "ab cdtimeout", "ab\tcdtimeout", "abvcdtimeout", "unrelated"}, + {true, true, true, true, false, false, false, false}, true); + check_recall(R"(ab[\a]cdtimeout)", {"ab\acdtimeout", "abacdtimeout", "unrelated"}, + {true, false, false}, true); + check_recall(R"(ab[\f]cdtimeout)", {"ab\fcdtimeout", "abfcdtimeout", "unrelated"}, + {true, false, false}, true); + check_recall(R"(ab[\v]cdtimeout)", + {"ab\vcdtimeout", "ab\ncdtimeout", "ab\rcdtimeout", "ab\fcdtimeout", + "ab cdtimeout", "ab\tcdtimeout", "abvcdtimeout", "unrelated"}, + {true, true, true, true, false, false, false, false}, true); + + check_recall(R"(ab\141cdtimeout)", {"abacdtimeout", "ab141cdtimeout", "unrelated"}, + {true, false, false}, true); + check_recall(R"(ab[\141]cdtimeout)", {"abacdtimeout", "ab1cdtimeout", "unrelated"}, + {true, false, false}, true); + + // Hyperscan accepts \Z even though RE2 rejects it. It must not become a literal Z. + check_recall(R"(timeout\Z)", {"timeout", "timeoutx", "unrelated"}, {true, false, false}, false); + + // An escape outside the gram parser's subset may disable the entire query. Hyperscan's + // horizontal whitespace class must not be mistaken for the literal h. + check_recall(R"(ab\hcdtimeout)", + {"ab cdtimeout", "ab\tcdtimeout", "ab\ncdtimeout", "abhcdtimeout", "unrelated"}, + {true, true, false, false, false}, false); + + // Known zero-width escapes and escaped punctuation retain their literal constraints. + check_recall(R"(\Atimeout\z)", {"timeout", "xtimeout", "unrelated"}, {true, false, false}, + true); + check_recall(R"(\btimeout\b)", {"timeout", "xtimeoutx", "unrelated"}, {true, false, false}, + true); + check_recall(R"(\Btimeout\B)", {"xtimeoutx", "timeout", "unrelated"}, {true, false, false}, + true); + check_recall(R"(ab\.cdtimeout)", {"ab.cdtimeout", "abxcdtimeout", "unrelated"}, + {true, false, false}, true); +} + +TEST(RegexGramRecallTest, GroupFlagsPreserveScalarRecall) { + // Disabling case folding inside any group must restore the enclosing (?i) after ')'. + // The final case-sensitive suffix keeps every scheme selective even if the folded part + // has too many alternatives to enumerate. + for (const auto* pattern : + {"(?i)(?-i:abc)DEF(?-i:timeout)", "(?i)((?-i)abc)DEF(?-i:timeout)", + "(?i)(?:(?-i)abc)DEF(?-i:timeout)", "(?i)(?P(?-i)abc)DEF(?-i:timeout)", + "(?i)(?:(?-i)(abc))DEF(?-i:timeout)"}) { + check_recall(pattern, {"abcdeftimeout", "abcDEFtimeout", "ABCdeftimeout", "unrelated"}, + {true, true, false, false}, true); + } + + // A flags-only group changes the remainder of its enclosing scope; it is not a nested + // scope whose flags should be restored immediately after its own closing parenthesis. + check_recall("(?i)ABC(?-i)timeout", {"abctimeout", "ABCtimeout", "ABCTIMEOUT", "unrelated"}, + {true, true, false, false}, true); + check_recall("(?:(?i)ABC(?-i)timeout)END", + {"abctimeoutEND", "ABCtimeoutEND", "ABCTIMEOUTEND", "abctimeoutend", "unrelated"}, + {true, true, false, false, false}, true); +} + +TEST(RegexGramRecallTest, QuotedLiteralQuantifiersPreserveScalarRecall) { + // Quoting changes how characters are tokenized; it does not group them into one atom. + check_recall(R"(\Qabc\E+deftimeout)", + {"abcccdeftimeout", "abcdeftimeout", "abdeftimeout", "unrelated"}, + {true, true, false, false}, true); + check_recall(R"(\Qab\E{2}timeout)", {"abbtimeout", "ababtimeout", "abtimeout", "unrelated"}, + {true, false, false, false}, true); + check_recall(R"(\Qabc\E*timeout)", {"abtimeout", "abccctimeout", "atimeout", "unrelated"}, + {true, true, false, false}, true); + check_recall(R"(\Qa.b+\E{2}timeout)", + {"a.b++timeout", "a.b+timeout", "abbbtimeout", "unrelated"}, + {true, false, false, false}, true); + + // Explicit parentheses still make the whole quoted sequence the repeated atom. + check_recall(R"((\Qabc\E)+timeout)", + {"abcabctimeout", "abctimeout", "abccctimeout", "unrelated"}, + {true, true, false, false}, true); + + // Empty quotes are transparent, including between an existing atom and its quantifier. + check_recall(R"(ab\Q\E+timeout)", {"abbbtimeout", "abtimeout", "atimeout", "unrelated"}, + {true, true, false, false}, true); + check_recall(R"((ab)\Q\E{2}timeout)", {"ababtimeout", "abbtimeout", "unrelated"}, + {true, false, false}, true); + check_recall(R"(ab\Q\E\Q\E{2}timeout)", {"abbtimeout", "abtimeout", "unrelated"}, + {true, false, false}, true); + check_recall(R"(\Q\Etimeout\Q\E)", {"timeout", "unrelated"}, {true, false}, true); + // Hyperscan rejects the two separated repeats. Its RE2 fallback treats the '?' after + // the empty quote as a second repeat over b+, allowing zero occurrences of b. Removing + // the quote would instead join '+?' into one lazy repeat, which still requires a b. + std::vector matches; + const auto hs_only = scalar_matches(R"(ab+\Q\E?timeout)", {"atimeout"}, &matches, false); + ASSERT_FALSE(hs_only.ok()); + EXPECT_NE(hs_only.to_string().find("Invalid repeat"), std::string::npos); + check_recall(R"(ab+\Q\E?timeout)", {"abbbtimeout", "abtimeout", "atimeout", "unrelated"}, + {true, true, true, false}, true); + check_recall(R"(ab+?timeout)", {"abbbtimeout", "abtimeout", "atimeout", "unrelated"}, + {true, true, false, false}, true); + + // The final atom is an entire Unicode code point, including a four-byte character. + check_recall(R"(\Qabé\E{2}timeout)", {"abéétimeout", "abétimeout", "unrelated"}, + {true, false, false}, true); + check_recall(R"(\Qab😀\E{2}timeout)", {"ab😀😀timeout", "ab😀timeout", "unrelated"}, + {true, false, false}, true); +} + +TEST(RegexGramRecallTest, EmbeddedNulPreservesScalarRecall) { + // These patterns bypass the scalar string fast paths. Hyperscan receives a C string, + // so the literal constraints after the raw NUL are not required by the scalar match. + check_recall(std::string("foo[0-9]") + '\0' + "timeout", {"foo1", "xfoo2x", "foo", "unrelated"}, + {true, true, false, false}, false); + check_recall(std::string("%prefix") + '\0' + "%timeout%", + {"prefix", "xprefixx", "timeout", "unrelated"}, + {true, true, false, false}, false); + check_recall(std::string("prefix_") + '\0' + "%timeout%", + {"prefix1", "prefix12", "xprefix1", "prefix", "unrelated"}, + {true, true, false, false, false}, false); + + // The scalar string fast paths are length-aware. Raw NUL must not be universally + // reinterpreted as a terminator: these patterns still require their entire literal. + const std::string literal = std::string("foo") + '\0' + "timeout"; + check_recall(literal, {literal, "foo", "unrelated"}, {true, false, false}, false); + check_recall(literal, {literal, "foo", "unrelated"}, {true, false, false}, false); + + // An escaped NUL is regex syntax, not a raw pattern terminator. Its determined suffix + // and ordinary LIKE segments must continue to filter unrelated rows. + check_recall(R"(foo\x00timeout)", {literal, "foo", "unrelated"}, {true, false, false}, true); + check_recall("%prefix%timeout%", {"prefixtimeout", "prefix", "unrelated"}, + {true, false, false}, true); +} + +} // namespace doris::segment_v2::gram From 789365d33f55621e38a192d41f43b5e51ce492e1 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Thu, 10 Sep 2026 20:55:19 +0800 Subject: [PATCH 03/36] [feature](index) gram-family analyzers and the SNII write path: docs-only grams, per-segment density, stop-gram and the high-df digest Write side of the gram index, on the SNII storage format only. - The built-in ngram tokenizer gains mode=sparse|dense|auto; with a mode it becomes a GramTokenizer. An analyzer is gram family only when its tokenizer is such an ngram and it carries no char filter or token filter (any filter breaks "stored term == extractor(raw value)"), and the writer forces such an index docs-only. The tokenizer is reused across rows and its retained buffers are charged to the writer's memory reporter. - Three decisions per segment, taken from the segment's own rows and recorded with it in SniiCoreMetadataPB.gram_scheme / high_df_terms: * the density is solved from the first 4 MB of the column (enable_gram_index_adaptive_density, gram_index_min_literal_bytes, gram_index_density_coverage_permille, gram_index_density_sample_bytes); the tokenizer's density is the fallback; * stop-gram: a gram present in more than 0.15% of the segment's rows keeps its dictionary entry and df but its posting list is not written (DictEntry::posting_dropped, dict flag bit 5). It reads as "every row", which is what the cost gate would have decided for it anyway, so LIKE/REGEXP lose nothing while the index sheds 80-92% of its bytes. Query types that need exact postings are guarded at the planner and at the reader and fall back to row-level evaluation. Segments below 2,000 rows keep every posting; there is no switch; * a high-df digest of the most common terms (hash, df) plus the ceiling every unrecorded term sits below, so the query side can bound a candidate count before reading a dictionary block. - A dropped posting is df-only in the entry body; the dict entry, dict block and frq layouts are otherwise the released ones. Tests: tokenizer and custom-analyzer detection, writer gram family and docs-only forcing, core metadata and dict entry round trips including the dropped flag, the stop-gram writer and its cut-off curve, digest gating and its cost, scheme inheritance through CREATE INDEX / BUILD INDEX, and the proto field numbers of the two new metadata fields. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01METuP3aVRn8dfCU62PivnF --- be/src/common/config.cpp | 33 ++ be/src/common/config.h | 54 ++ be/src/storage/index/index_file_writer.cpp | 82 +++ be/src/storage/index/index_file_writer.h | 3 + .../inverted/analyzer/analyzer_provider.cpp | 34 ++ .../inverted/analyzer/analyzer_provider.h | 19 + .../inverted/analyzer/custom_analyzer.cpp | 27 +- .../index/inverted/analyzer/custom_analyzer.h | 11 +- .../tokenizer/ngram/gram_tokenizer.cpp | 55 ++ .../inverted/tokenizer/ngram/gram_tokenizer.h | 64 ++ .../ngram/ngram_tokenizer_factory.cpp | 34 ++ .../tokenizer/ngram/ngram_tokenizer_factory.h | 23 + .../index/snii/format/core_metadata.cpp | 97 +++ .../storage/index/snii/format/core_metadata.h | 46 ++ .../storage/index/snii/format/dict_entry.cpp | 17 +- be/src/storage/index/snii/format/dict_entry.h | 14 +- .../index/snii/format/format_constants.h | 36 +- .../index/snii/query/docid_conjunction.cpp | 15 + .../index/snii/query/gram_boolean_query.cpp | 552 ++++++++++++++++++ .../index/snii/query/gram_boolean_query.h | 175 ++++++ .../snii/reader/logical_index_reader.cpp | 10 + .../index/snii/reader/logical_index_reader.h | 12 + .../storage/index/snii/snii_index_writer.cpp | 270 ++++++++- be/src/storage/index/snii/snii_index_writer.h | 69 ++- .../snii/writer/logical_index_writer.cpp | 148 ++++- .../index/snii/writer/logical_index_writer.h | 32 + .../ananlyzer/custom_analyzer_test.cpp | 62 ++ .../tokenizer/gram_tokenizer_test.cpp | 142 +++++ .../index/snii/format/core_metadata_test.cpp | 70 +++ .../index/snii/format/dict_entry_test.cpp | 70 +++ .../format/gram_index_section_sizes_test.cpp | 236 ++++++++ .../snii/format/gram_stop_gram_curve_test.cpp | 318 ++++++++++ .../format/snii_proto_field_numbers_test.cpp | 4 +- .../writer/gram_density_calibration_test.cpp | 204 +++++++ .../snii/writer/high_df_digest_cost_test.cpp | 178 ++++++ .../snii/writer/snii_gram_inherit_test.cpp | 354 +++++++++++ .../index/snii/writer/stop_gram_test.cpp | 329 +++++++++++ be/test/storage/index/snii_writer_test.cpp | 393 +++++++++++++ build-support/check-header-deps.py | 10 +- gensrc/proto/snii.proto | 55 ++ 40 files changed, 4329 insertions(+), 28 deletions(-) create mode 100644 be/src/storage/index/inverted/analyzer/analyzer_provider.cpp create mode 100644 be/src/storage/index/inverted/tokenizer/ngram/gram_tokenizer.cpp create mode 100644 be/src/storage/index/inverted/tokenizer/ngram/gram_tokenizer.h create mode 100644 be/src/storage/index/snii/query/gram_boolean_query.cpp create mode 100644 be/src/storage/index/snii/query/gram_boolean_query.h create mode 100644 be/test/storage/index/inverted/tokenizer/gram_tokenizer_test.cpp create mode 100644 be/test/storage/index/snii/format/gram_index_section_sizes_test.cpp create mode 100644 be/test/storage/index/snii/format/gram_stop_gram_curve_test.cpp create mode 100644 be/test/storage/index/snii/writer/gram_density_calibration_test.cpp create mode 100644 be/test/storage/index/snii/writer/high_df_digest_cost_test.cpp create mode 100644 be/test/storage/index/snii/writer/snii_gram_inherit_test.cpp create mode 100644 be/test/storage/index/snii/writer/stop_gram_test.cpp diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 85763ed7a92868..8f84bdc79cffa9 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1346,6 +1346,39 @@ double get_inverted_index_candidate_pushdown_ratio() { // inverted index match bitmap cache size DEFINE_String(inverted_index_query_cache_limit, "10%"); +namespace { + +bool valid_gram_index_candidate_ratio(int32_t value) { + return value >= 0; +} + +bool valid_gram_index_candidate_min_rows(int32_t value) { + return value >= 0; +} + +} // namespace + +// Whether LIKE/REGEXP tries to compile a constant pattern into a gram boolean query pushed down +// to a gram-family inverted index (master switch). +DEFINE_mBool(enable_gram_index_regexp, "true"); + +// Cost gate for the gram boolean query: give up pruning above this share (in basis points) of a +// segment's rows, and only on segments of at least this many rows. Giving up returns the segment's +// whole docid range, so it can never drop a candidate row. +DEFINE_mInt32(gram_index_max_candidate_ratio_bp, "15"); +DEFINE_Validator(gram_index_max_candidate_ratio_bp, valid_gram_index_candidate_ratio); +DEFINE_mInt32(gram_index_candidate_ratio_min_rows, "65536"); + +// On by default: a configured density is wrong on every column but the one it was chosen for, +// and the solve costs one pass over a bounded sample. +DEFINE_mBool(enable_gram_index_adaptive_density, "true"); +// 12 bytes at 95% of windows. Both are the promise rather than a tuning pair -- what a user +// may reasonably change is how short a literal they expect to find, not the boundary rate. +DEFINE_mInt32(gram_index_min_literal_bytes, "12"); +DEFINE_mInt32(gram_index_density_coverage_permille, "950"); +DEFINE_mInt64(gram_index_density_sample_bytes, "4194304"); +DEFINE_Validator(gram_index_candidate_ratio_min_rows, valid_gram_index_candidate_min_rows); + // condition cache limit DEFINE_Int16(condition_cache_limit, "512"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 63020e990a8465..ba08064378a7ad 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1391,6 +1391,60 @@ DECLARE_String(inverted_index_query_cache_limit); // it affects only writers created after the transition and never changes query/cache semantics. // Release-calibrated query-planner coefficients. Both remain mutable for controlled recalibration. +// Whether LIKE/REGEXP tries to compile a constant pattern into a gram boolean query pushed down +// to a gram-family inverted index (master switch). Turning it off behaves as if the index did +// not exist -- it only gives up the speedup, it never changes query results. +// +// A pattern is compiled against the gram scheme the segment itself carries, read back from the +// core metadata of the physical index the query is about to read, so the scheme can never +// disagree with the one the writer used -- a policy that was dropped and recreated with different +// properties does not affect segments already written. +DECLARE_mBool(enable_gram_index_regexp); + +// Cost gate for the gram boolean query, in basis points (1/10000) of a segment's rows: give up +// pruning once the candidate set is larger than that share. 0 disables the gate, 10000 or more can +// never fire. This is a COST switch, never a semantic one -- a gram index only produces a superset +// of the candidate rows and the expression above re-verifies each of them, so giving up means the +// query returns the whole docid range of the segment and prunes nothing. It can therefore only +// cost speed, never correctness, whatever value it is set to. +// +// The unit is basis points rather than percent because the useful range is far below one percent. +// Pruning only pays when the surviving candidates are sparse enough that whole pages of the column +// can be skipped: with B rows per remote read unit and candidates spread uniformly, the fraction of +// units still touched is 1 - (1 - C/N)^B, which is already ~1 by C/N = 1%. Measured on a 34.46M row +// httplogs table against remote object storage, cold-read speedup by candidate count was 14.1x at +// 2 candidates, 5.5x at 128, 1.5x at 25K, 1.2x at 46K, and turned into a regression from 56K +// upwards, bottoming out at 0.70x. 15 bp (0.15%) is the largest share that admitted no regression +// in that sweep while keeping every double-digit win. +DECLARE_mInt32(gram_index_max_candidate_ratio_bp); + +// Row floor below which the candidate ratio gate above is not applied at all. A small segment's +// entire gram index is a few KB and one or two requests, so giving up there saves nothing +// measurable while throwing away the pruning the index really does deliver; the ratio only starts +// to mean something at a size where the skipped index IO can outweigh the rows it stops +// eliminating. The default is one Roaring container's worth of rows (65536). 0 applies the ratio +// at every segment size. +DECLARE_mInt32(gram_index_candidate_ratio_min_rows); + +// Adaptive gram density: solve the boundary rate from each segment's own bytes instead of +// taking the rate configured on the tokenizer. +// +// A configured rate cannot mean the same thing on two columns. Measured at a nominal 0.25 the +// realised grams per byte were 0.204 on log text, 0.287 on URL paths and 0.182 on agent +// traces, and the coverage it bought ranged from 91.7% of 12-byte literals to 98.9% -- under +// serving one column while overpaying on another, in every posting list. Solving instead +// makes the promise the constant and the rate the variable. +DECLARE_mBool(enable_gram_index_adaptive_density); +// The promise the solve keeps, not a tuning pair: literals of at least this many bytes are +// findable, for this share of the column's own windows of that length. Both are dimensionless +// and the same on every dataset; what varies is the density they resolve to. +DECLARE_mInt32(gram_index_min_literal_bytes); +DECLARE_mInt32(gram_index_density_coverage_permille); +// How much of a segment is held back to solve on. The sample is buffered rather than +// tokenized, so this is a transient memory cost and a bound on how long the write path waits +// before it can cut anything; the histogram behind the solve is a fixed 256 KB regardless. +DECLARE_mInt64(gram_index_density_sample_bytes); + // condition cache limit DECLARE_Int16(condition_cache_limit); diff --git a/be/src/storage/index/index_file_writer.cpp b/be/src/storage/index/index_file_writer.cpp index 923a39dda508f1..f6722ddaee6ca8 100644 --- a/be/src/storage/index/index_file_writer.cpp +++ b/be/src/storage/index/index_file_writer.cpp @@ -37,6 +37,7 @@ #include "storage/index/inverted/inverted_index_desc.h" #include "storage/index/inverted/inverted_index_fs_directory.h" #include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/snii/query/gram_boolean_query.h" #include "storage/index/snii/snii_blob_staging_directory.h" #include "storage/index/snii/snii_doris_adapter.h" #include "storage/tablet/tablet_schema.h" @@ -44,6 +45,85 @@ namespace doris::segment_v2 { +// Resolves the stop-gram df threshold for one index, in documents. Three conditions must +// all hold, and every one of them is a correctness or safety condition rather than a +// tuning choice: +// +// - gram family only. Dropping a posting turns its term into one matching every +// document. LIKE/REGEXP survives that because its candidates are re-checked against +// the column; an ordinary full-text index has no such re-check and would return rows +// that do not match. +// - docs-only. Dropping the posting drops its positions with it, so any index that +// stores positions would lose phrase capability for that term. Gram indexes are +// written docs-only, so this is an assertion of that rather than a restriction. +// It carries a second guarantee worth stating: the compaction merge fast path +// requires positions (snii/compaction/eligibility.cpp validate_source_shape), so a +// segment that can hold dropped postings can never be a merge source. The merge +// therefore never has to reconstruct a posting list that was never written -- it +// rebuilds such an index from the raw column instead, which recomputes stop-gram +// against the merged segment's own size. Keep these two conditions opposed: making +// either side accept both shapes would let the merge silently emit a term with a +// real df and an empty posting list, which reads as "matches nothing". +// - above the row floor. A small segment's whole gram index is a few KB; there is +// nothing to save and the pruning it does deliver is worth keeping. +// +// There is no switch. Dropping is what lets a gram index meet its size budget of index +// bytes <= 0.30 of the compressed column data: at density 0.25 the terms above the line are +// under 1% of the vocabulary but hold ~85% of the posting entries, and dropping them takes +// textbench from 1.097 to 0.218 index/data, httplogs from 0.865 to 0.071 and agentlogs from +// 0.853 to 0.094, with recall unchanged and the rare-literal speedups intact. What it costs +// is filtering for literals made entirely of grams above the line (0.15% of the segment); +// the cost gate would have refused most of those anyway. A dropped posting comes back only +// when the index is rebuilt (compaction, schema change), so a per-column opt-out, if one is +// ever needed, belongs in the index properties where it is recorded with the schema -- not +// in a BE-wide runtime flag that reaches only the segments written after it flips. +// +// NOT in the anonymous namespace: the UT pins this policy directly. +uint32_t snii_effective_stop_gram_df_threshold(const doris::snii::writer::SniiIndexInput& input) { + if (!input.gram_scheme.has_value() || doris::snii::format::has_positions(input.config)) { + return 0; + } + // A segment with fewer rows than the digest's divisor has no meaningful notion of a + // "common" term: its floor would have to be clamped up to 1, at which point dropping + // postings for df of 4 in a 500-row segment is pure loss. The reader agrees by the same + // arithmetic -- its digest ceiling lands at 0 there and its gate falls back rather than + // deriving a budget -- so both sides stand down together. + if (input.doc_count < doris::snii::format::kHighDfDigestDivisor) { + return 0; + } + // The threshold is derived, not configured: it is the same line the query side's cost gate + // draws from the high-df digest -- the digest's floor times the budget multiple, which + // lands at 0.15% of the segment. Writing and reading agree by construction, and what the + // writer drops is what a reader would have refused to read. + // + // This line is chosen for size, and the cost is known and accepted. Measured across the + // four corpora, it drops 88.6% of all posting entries and takes index/data from 1.097 to + // 0.218 on log text; a tenth-of-the-documents line (tau = 0.10, the value the FREE line of + // work uses) drops 14.7% and lands at 0.924, which does not meet the size target the index + // is held to. + // + // What it costs is every query whose RAREST gram sits above the line. Two groups do: + // + // - all of CJK, back when it was indexed one term per code point: the terms a query is + // made of are ordinary characters sitting in a few percent of the rows. Measured on + // weibo, every pattern tried lost its filtering entirely: a four-character literal + // proposed 500,000 candidates for its 18 matching rows and eliminated none, against + // 148 candidates and 499,852 rows eliminated on the same corpus indexed without + // stop-gram. It is one of the reasons non-ASCII text is no longer indexed at all + // (see gram_extractor.cpp). + // - ASCII patterns in the middle of the selectivity range. `/history/images/` matches + // 0.73% of httplogs and goes the same way: 70,976 candidates become 3,000,000, with + // nothing eliminated. + // + // What survives is the needle: a pattern whose grams sit orders of magnitude below the + // line keeps every bit of its pruning, and those are the queries with the large wins -- + // 13 rows out of 3,000,000 still runs 121x faster than a scan on a cold segment. The line + // therefore keeps the extremes and gives up the middle, which measured 1.2x-2.4x. + const uint64_t floor = + static_cast(input.doc_count) / doris::snii::format::kHighDfDigestDivisor; + return static_cast(floor * doris::snii::query::kCandidateBudgetCeilingMultiple); +} + // Shared write-parameter resolution for one SNII index flush; `input->config` // must already be set. BOTH the build path (add_snii_index) and the T2.2 // compaction-merge streamed session resolve through this single helper, so the @@ -70,6 +150,7 @@ void snii_resolve_index_write_params(bool is_direct_load, bool has_norms, input->target_dict_block_bytes = static_cast(config::snii_target_dict_block_bytes); } + input->stop_gram_df_threshold = snii_effective_stop_gram_df_threshold(*input); } IndexFileWriter::IndexFileWriter(io::FileSystemSPtr fs, std::string index_path_prefix, @@ -268,6 +349,7 @@ Status IndexFileWriter::add_snii_index(const TabletIndex* index_meta, uint32_t d input.doc_count = doc_count; input.null_docids = std::move(null_docids); input.encoded_norms = std::move(options.encoded_norms); + input.gram_scheme = options.gram_scheme; input.term_source = term_buffer; input.mem_reporter = mem_reporter; snii_resolve_index_write_params(options.is_direct_load, !input.encoded_norms.empty(), &input); diff --git a/be/src/storage/index/index_file_writer.h b/be/src/storage/index/index_file_writer.h index 3a0346b95d428a..14257a5a2c40a2 100644 --- a/be/src/storage/index/index_file_writer.h +++ b/be/src/storage/index/index_file_writer.h @@ -43,6 +43,7 @@ #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "storage/index/index_storage_format.h" +#include "storage/index/inverted/gram/gram_scheme.h" #include "storage/index/inverted/inverted_index_common.h" #include "storage/index/inverted/inverted_index_compound_reader.h" #include "storage/index/inverted/inverted_index_searcher.h" @@ -110,6 +111,8 @@ class IndexFileWriter { // the prx region compresses at snii_prx_zstd_level_direct_load; // compaction / schema change / ADD INDEX keep snii_prx_zstd_level. bool is_direct_load = false; + // The exact gram tokenizer scheme that produced this index's dictionary. + std::optional gram_scheme; // One byte of BM25 norms per document; empty for keyword or positionless indexes. // If nonempty, its size must equal doc_count, and postings retain frequencies for scoring. std::vector encoded_norms; diff --git a/be/src/storage/index/inverted/analyzer/analyzer_provider.cpp b/be/src/storage/index/inverted/analyzer/analyzer_provider.cpp new file mode 100644 index 00000000000000..add57552438887 --- /dev/null +++ b/be/src/storage/index/inverted/analyzer/analyzer_provider.cpp @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/analyzer/analyzer_provider.h" + +// gram_scheme.h is completely defined only in this .cpp: to keep the forward include closure of +// exec/pipeline/dependency.h under budget, analyzer_provider.h only forward-declares GramScheme +// (see the comment at the top of analyzer_provider.h). The default implementation of +// gram_scheme() has to construct an empty std::optional, which requires the complete +// type, so it was moved here, where that definition is needed only while this translation unit +// is actually compiled. +#include "storage/index/inverted/gram/gram_scheme.h" + +namespace doris::segment_v2::inverted_index { + +std::optional AnalyzerProvider::gram_scheme() const { + return std::nullopt; +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/analyzer/analyzer_provider.h b/be/src/storage/index/inverted/analyzer/analyzer_provider.h index 295daaca37efc4..6c0cfc05bad38e 100644 --- a/be/src/storage/index/inverted/analyzer/analyzer_provider.h +++ b/be/src/storage/index/inverted/analyzer/analyzer_provider.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include "storage/index/inverted/abstract_analysis_factory.h" @@ -26,12 +27,30 @@ namespace lucene::analysis { class Analyzer; } +// Forward declaration only, no #include of gram_scheme.h: this header sits on the +// tablet_schema.h -> inverted_index_parser.h path that almost every TU pulls in (the forward +// include closure budget of exec/pipeline/dependency.h has long been pinned at 357), and +// gram_scheme.h, small as it is, would still push that closure over budget. The places that +// really need the complete definition (the _gram_scheme member of CustomAnalyzerProvider and the +// actual implementation of gram_scheme()) include it in custom_analyzer.h and +// analyzer_provider.cpp respectively. +namespace doris::segment_v2::gram { +struct GramScheme; +} + namespace doris::segment_v2::inverted_index { class AnalyzerProvider { public: virtual ~AnalyzerProvider() = default; virtual std::shared_ptr get_analyzer() const = 0; + // Gram-family detection: only when the tokenizer is ngram and carries a "mode" property + // (sparse|dense|auto) does CustomAnalyzerProvider override this to return a populated + // GramScheme; every other provider (built-in parsers, normalizers, ...) keeps the default + // implementation in analyzer_provider.cpp, which always returns nullopt. That default lives + // in the .cpp instead of being inlined here because GramScheme is only forward-declared at + // this point (see the comment above). + virtual std::optional gram_scheme() const; }; using AnalyzerProviderPtr = std::shared_ptr; diff --git a/be/src/storage/index/inverted/analyzer/custom_analyzer.cpp b/be/src/storage/index/inverted/analyzer/custom_analyzer.cpp index ec1298bf411652..9c8b65e7108897 100644 --- a/be/src/storage/index/inverted/analyzer/custom_analyzer.cpp +++ b/be/src/storage/index/inverted/analyzer/custom_analyzer.cpp @@ -24,6 +24,7 @@ #include "runtime/exec_env.h" #include "storage/index/inverted/analysis_factory_mgr.h" #include "storage/index/inverted/token_stream.h" +#include "storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h" namespace doris::segment_v2::inverted_index { namespace {} // namespace @@ -103,7 +104,31 @@ CustomAnalyzerPtr CustomAnalyzer::build_custom_analyzer( CustomAnalyzerProvider::CustomAnalyzerProvider( ImmutableCustomAnalyzerConfigPtr config, std::map /*outer_char_filter_map*/) - : _config(std::move(config)), _analyzer(CustomAnalyzer::build_custom_analyzer(_config)) {} + : _config(std::move(config)), _analyzer(CustomAnalyzer::build_custom_analyzer(_config)) { + // Gram-family detection: when the tokenizer is "ngram", hand its Settings to + // NGramTokenizerFactory::parse_gram_scheme so the same property mapping is reused (R16, + // DRY). build_custom_analyzer() has already constructed that tokenizer successfully above + // (otherwise it would have thrown and aborted construction), so re-parsing the same + // configuration here cannot fail; should it fail anyway, we fall back to nullopt rather than + // rethrowing. + // + // R22 (fail-safe): an analyzer carrying any char filter or token filter is never treated as + // gram family. The whole value of the gram family rests on the row invariant "the stored + // term == GramExtractor.extract(raw column value)", which the query side (phase C) uses to + // rewrite a regex into a conjunction of grams; a char filter rewrites the text and a token + // filter rewrites, adds or removes grams, and either one breaks that equality. Better to + // fall back to a full scan than to drop rows on an invariant that no longer holds. + const bool has_filters = !_config->get_char_filter_configs().empty() || + !_config->get_token_filter_configs().empty(); + if (const auto& tokenizer_config = _config->get_tokenizer_config(); + !has_filters && tokenizer_config != nullptr && tokenizer_config->get_name() == "ngram") { + if (Status st = NGramTokenizerFactory::parse_gram_scheme(tokenizer_config->get_params(), + &_gram_scheme); + !st.ok()) { + _gram_scheme.reset(); + } + } +} void CustomAnalyzer::Builder::with_tokenizer(const std::string& name, const Settings& params) { _tokenizer = AnalysisFactoryMgr::instance().create(name, params); } diff --git a/be/src/storage/index/inverted/analyzer/custom_analyzer.h b/be/src/storage/index/inverted/analyzer/custom_analyzer.h index 28345a60c30186..b3e0dc3c9083f1 100644 --- a/be/src/storage/index/inverted/analyzer/custom_analyzer.h +++ b/be/src/storage/index/inverted/analyzer/custom_analyzer.h @@ -24,6 +24,7 @@ #include "storage/index/inverted/analyzer/analyzer_provider.h" #include "storage/index/inverted/analyzer/custom_analyzer_config.h" #include "storage/index/inverted/char_filter/char_filter_factory.h" +#include "storage/index/inverted/gram/gram_scheme.h" #include "storage/index/inverted/setting.h" #include "storage/index/inverted/token_filter/token_filter_factory.h" #include "storage/index/inverted/tokenizer/tokenizer_factory.h" @@ -84,10 +85,18 @@ class CustomAnalyzerProvider final : public AnalyzerProvider { std::map outer_char_filter_map = {}); std::shared_ptr get_analyzer() const override { return _analyzer; } + // Computed once by the constructor from the analyzer configuration and cached: it holds a + // value when the tokenizer is "ngram" with a non-empty "mode" property and the analyzer + // carries no char filter or token filter, and is nullopt otherwise (R22 fail-safe: any + // filter invalidates the row invariant "the stored term == GramExtractor.extract(raw column + // value)"). The value is computed by NGramTokenizerFactory::parse_gram_scheme, keeping a + // single source of truth with the tokenizer's own property parsing. + std::optional gram_scheme() const override { return _gram_scheme; } private: ImmutableCustomAnalyzerConfigPtr _config; std::shared_ptr _analyzer; + std::optional _gram_scheme; }; -} // namespace doris::segment_v2::inverted_index \ No newline at end of file +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/tokenizer/ngram/gram_tokenizer.cpp b/be/src/storage/index/inverted/tokenizer/ngram/gram_tokenizer.cpp new file mode 100644 index 00000000000000..25dcf6b4743765 --- /dev/null +++ b/be/src/storage/index/inverted/tokenizer/ngram/gram_tokenizer.cpp @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/tokenizer/ngram/gram_tokenizer.h" + +#include "common/logging.h" + +namespace doris::segment_v2::inverted_index { + +void GramTokenizer::reset() { + DorisTokenizer::reset(); + _grams.clear(); + _next = 0; + _char_buffer = nullptr; + _char_length = 0; + if (_in == nullptr || _in->size() == 0) { + return; + } + // Same as NGramTokenizer::reset(): read the whole input in one go + // (ngram_tokenizer.cpp:83-94). + _char_length = _in->read(reinterpret_cast(&_char_buffer), 0, + static_cast(_in->size())); + // The read must cover everything: one byte short means a handful of grams fewer, which + // breaks the invariant "the grams stored in the index are a superset of the grams the query + // side needs" outright (rows go missing with no way to notice). + DCHECK_EQ(_char_length, static_cast(_in->size())); + if (_char_length <= 0 || _char_buffer == nullptr) { + return; + } + _extractor.extract(std::string_view(_char_buffer, _char_length), &_grams); +} + +Token* GramTokenizer::next(Token* token) { + if (_next >= _grams.size()) { + return nullptr; + } + set(token, _grams[_next++], 1); + return token; +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/tokenizer/ngram/gram_tokenizer.h b/be/src/storage/index/inverted/tokenizer/ngram/gram_tokenizer.h new file mode 100644 index 00000000000000..9e48a52b2f0acc --- /dev/null +++ b/be/src/storage/index/inverted/tokenizer/ngram/gram_tokenizer.h @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include +#include + +#include "storage/index/inverted/gram/gram_extractor.h" +#include "storage/index/inverted/tokenizer/tokenizer.h" + +namespace doris::segment_v2::inverted_index { + +// Adapts gram::GramExtractor to the DorisTokenizer interface: one column value = one reset, with +// every gram extracted at once. Unlike NGramTokenizer, which slides a window byte by byte, there +// is no incremental window state here: reset() reads the reader's entire input in one go and +// hands it to GramExtractor, and next() just yields the results in order. +class GramTokenizer : public DorisTokenizer { +public: + explicit GramTokenizer(const gram::GramScheme& scheme) : _extractor(scheme) {} + ~GramTokenizer() override = default; + + Token* next(Token* token) override; + void reset() override; + const gram::GramScheme& scheme() const { return _extractor.scheme(); } + + // Retunes the boundary rate before any row of a segment is cut. See + // GramExtractor::set_density_permille for why it may only happen there. + void set_density_permille(uint16_t density_permille) { + _extractor.set_density_permille(density_permille); + } + + // Bytes of buffer capacity this tokenizer keeps between rows: the gram list plus the + // extractor's scratch. Both are sized by the longest row seen so far and are never shrunk, + // so this is a steady resident cost rather than a per-row spike -- which is exactly the shape + // SNII's MemoryReporter wants (SniiIndexColumnWriter mirrors it, delta-charged, after every + // row). It only means anything while the tokenizer outlives the row, i.e. when the writer + // obtained it from reusableTokenStream. + size_t reserved_bytes() const { + return _grams.capacity() * sizeof(std::string_view) + _extractor.reserved_bytes(); + } + +private: + gram::GramExtractor _extractor; + const char* _char_buffer = nullptr; + int32_t _char_length = 0; + std::vector _grams; // views into _char_buffer or the extractor's fold copy + size_t _next = 0; +}; + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.cpp b/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.cpp index 982ba376d29e24..3c9f7076cb053a 100644 --- a/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.cpp +++ b/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.cpp @@ -17,13 +17,47 @@ #include "storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h" +#include + #include "common/exception.h" namespace doris::segment_v2::inverted_index { std::unordered_map NGramTokenizerFactory::MATCHERS; +Status NGramTokenizerFactory::parse_gram_scheme(const Settings& settings, + std::optional* out) { + out->reset(); + // The presence of "mode" (sparse|dense|auto) switches to the gram family, which is mutually + // exclusive with the legacy ngram sliding window; parsing, validation and defaults of the + // gram scheme are all delegated to GramScheme::from_properties, and this function only + // forwards the tokenizer properties. Absent min/max_gram come from GramScheme's own member + // initializers (3/16) and are deliberately not duplicated here -- two sets of defaults would + // drift sooner or later, and there can be only one source of truth. + if (settings.get_string("mode").empty()) { + return Status::OK(); + } + std::map props; + for (const auto& [k, v] : settings.sorted_entries()) { + props.emplace(k, v); + } + gram::GramScheme scheme; + RETURN_IF_ERROR(gram::GramScheme::from_properties(props, &scheme)); + *out = scheme; + return Status::OK(); +} + void NGramTokenizerFactory::initialize(const Settings& settings) { + std::optional scheme; + Status st = parse_gram_scheme(settings, &scheme); + if (!st.ok()) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "ngram tokenizer: {}", st.to_string()); + } + if (scheme.has_value()) { + _gram_scheme = scheme; + return; // skip the legacy max-min>1 validation and token_chars parsing + } + _min_gram = settings.get_int("min_gram", NGramTokenizer::DEFAULT_MIN_NGRAM_SIZE); _max_gram = settings.get_int("max_gram", NGramTokenizer::DEFAULT_MAX_NGRAM_SIZE); if (_min_gram <= 0 || _max_gram <= 0) { diff --git a/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h b/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h index 8b91369b51d4f2..e74eac55356692 100644 --- a/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h +++ b/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h @@ -17,8 +17,13 @@ #pragma once +#include + +#include "common/status.h" +#include "storage/index/inverted/gram/gram_scheme.h" #include "storage/index/inverted/setting.h" #include "storage/index/inverted/tokenizer/ngram/char_matcher.h" +#include "storage/index/inverted/tokenizer/ngram/gram_tokenizer.h" #include "storage/index/inverted/tokenizer/ngram/ngram_tokenizer.h" #include "storage/index/inverted/tokenizer/tokenizer_factory.h" @@ -37,6 +42,9 @@ class NGramTokenizerFactory : public TokenizerFactory { void initialize(const Settings& settings) override; TokenizerPtr create() override { + if (_gram_scheme.has_value()) { + return std::make_shared(*_gram_scheme); + } if (_matcher == nullptr) { return std::make_shared(_min_gram, _max_gram); } else { @@ -59,15 +67,30 @@ class NGramTokenizerFactory : public TokenizerFactory { return PositionCapability::kAlwaysUnitIncrement; } + // Returns the scheme for the gram family (the tokenizer properties contain "mode"), and + // nothing for a legacy ngram (no "mode"), so that Tasks 8/12 can tell whether the current + // tokenizer is in the gram family and obtain its scheme parameters. + std::optional gram_scheme() const { return _gram_scheme; } + static void initialize_matchers(); static CharMatcherPtr parse_token_chars(const Settings& settings); + // Map tokenizer Settings to a GramScheme: this is the single source of truth for that + // mapping, and both initialize() and CustomAnalyzerProvider (the user of + // gram/gram_family.h) must call it rather than each keeping its own copy of the property + // parsing (R16, DRY). When "mode" is absent, *out is set to nullopt and OK is returned + // (legacy ngram); on an illegal value (an unknown mode, an out-of-range min/max_gram, ...) + // InvalidArgument is returned and *out stays nullopt. Every key that is absent falls back to + // GramScheme's own member initializer (min_gram=3, max_gram=16, ...). + static Status parse_gram_scheme(const Settings& settings, std::optional* out); + private: static std::unordered_map MATCHERS; int32_t _min_gram = 0; int32_t _max_gram = 0; CharMatcherPtr _matcher; + std::optional _gram_scheme; // set when "mode" is present: create() goes gram }; }; // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/snii/format/core_metadata.cpp b/be/src/storage/index/snii/format/core_metadata.cpp index 0f6ef8f5a59cdd..2ef8a25321fd3f 100644 --- a/be/src/storage/index/snii/format/core_metadata.cpp +++ b/be/src/storage/index/snii/format/core_metadata.cpp @@ -27,6 +27,61 @@ #include "storage/index/snii/encoding/section_framer.h" namespace doris::snii::format { + +// Storage is a unity build (several .cpp files merged into one unity_N_cxx.cxx TU): other .cpp +// files in this directory may define file-level helpers with the same names, so the gram_scheme +// codec helpers go into this file-private namespace (with an inner anonymous namespace to keep +// internal linkage) instead of sharing the anonymous namespace further below, which keeps +// same-named symbols from different files from colliding in the unity TU (Ruling R8). +namespace core_metadata_detail { +namespace { + +void encode_gram_scheme(const segment_v2::gram::GramScheme& scheme, + doris::snii::SniiGramSchemePB* out) { + out->set_mode(static_cast(scheme.mode)); + out->set_min_len(scheme.min_len); + out->set_max_len(scheme.max_len); + out->set_density_permille(scheme.density_permille); + out->set_lower_case(scheme.lower_case); + out->set_hash_version(scheme.hash_version); +} + +Status decode_gram_scheme(const doris::snii::SniiGramSchemePB& input, + segment_v2::gram::GramScheme* out) { + if (input.mode() != 1 && input.mode() != 2) { + return Status::Error( + "core metadata: unsupported gram scheme mode {}", input.mode()); + } + const segment_v2::gram::GramScheme scheme { + .mode = static_cast(input.mode()), + .min_len = input.min_len(), + .max_len = input.max_len(), + .density_permille = input.density_permille(), + .lower_case = input.lower_case(), + .hash_version = input.hash_version()}; + // The valid range of each field is written down in exactly one place, + // GramScheme::from_properties (the single source of truth), so it is reused here through a + // "property round trip": a persisted scheme must round-trip back to the very same scheme, or + // the file counts as corrupted. Without this step a truncated (or tampered) PB would carry + // values such as min_len=0 all the way into GramExtractor -- every unset field of a partial + // message is 0, and 0 is not part of any valid scheme. + segment_v2::gram::GramScheme round_tripped; + const Status validated = + segment_v2::gram::GramScheme::from_properties(scheme.to_properties(), &round_tripped); + if (!validated.ok() || !(round_tripped == scheme)) { + return Status::Error( + "core metadata: invalid gram scheme (mode={}, min_len={}, max_len={}, " + "density_permille={}, hash_version={}): {}", + input.mode(), scheme.min_len, scheme.max_len, scheme.density_permille, + scheme.hash_version, validated.to_string()); + } + *out = scheme; + return Status::OK(); +} + +} // namespace +} // namespace core_metadata_detail + namespace { Status corrupted(std::string_view message) { @@ -106,6 +161,33 @@ Status decode_core_pb(const doris::snii::SniiCoreMetadataPB& input, CoreMetadata "core metadata: segment was written with CommonGrams, which is no longer " "supported; rebuild the index"); } + + if (input.has_gram_scheme()) { + segment_v2::gram::GramScheme gram_scheme; + RETURN_IF_ERROR( + core_metadata_detail::decode_gram_scheme(input.gram_scheme(), &gram_scheme)); + out->gram_scheme = gram_scheme; + } + + if (input.has_high_df_terms()) { + const auto& digest = input.high_df_terms(); + if (digest.term_hash_size() != digest.df_size()) { + return Status::Error( + "core metadata: high-df digest has {} hashes and {} frequencies", + digest.term_hash_size(), digest.df_size()); + } + out->high_df_terms.term_hash.assign(digest.term_hash().begin(), digest.term_hash().end()); + out->high_df_terms.df.assign(digest.df().begin(), digest.df().end()); + out->high_df_terms.df_ceiling = digest.df_ceiling(); + // The lookup is a binary search, so a digest that is not ascending would silently + // return wrong bounds rather than fail. Reject it instead: a wrong upper bound can + // make the gate give up on a query the index would have answered quickly. + if (!std::ranges::is_sorted(out->high_df_terms.term_hash)) { + return Status::Error( + "core metadata: high-df digest hashes are not ascending"); + } + } + // Norms encode BM25 document lengths in one byte and require positions for term frequencies. if (out->section_refs.norms.length != 0 && !has_positions(out->index_config)) { return corrupted("core metadata: norms require positions"); @@ -137,6 +219,21 @@ Status encode_core_metadata(const CoreMetadata& metadata, ByteSink* out) { } encode_region_ref(metadata.section_refs.null_bitmap, refs->mutable_null_bitmap()); encode_region_ref(metadata.section_refs.bsbf, refs->mutable_bsbf()); + if (metadata.gram_scheme.has_value()) { + core_metadata_detail::encode_gram_scheme(*metadata.gram_scheme, core.mutable_gram_scheme()); + } + // A ceiling with no entries is still worth writing: it says every term in this index is + // below it, which is the strongest bound the digest can offer. Absent BOTH means no + // digest was built at all (not a gram index, or a segment too small to have one), and + // the field stays off the wire so those segments are byte-identical to before. + if (!metadata.high_df_terms.empty() || metadata.high_df_terms.df_ceiling > 0) { + auto* digest = core.mutable_high_df_terms(); + digest->mutable_term_hash()->Assign(metadata.high_df_terms.term_hash.begin(), + metadata.high_df_terms.term_hash.end()); + digest->mutable_df()->Assign(metadata.high_df_terms.df.begin(), + metadata.high_df_terms.df.end()); + digest->set_df_ceiling(metadata.high_df_terms.df_ceiling); + } CoreMetadata validated; RETURN_IF_ERROR(decode_core_pb(core, &validated)); diff --git a/be/src/storage/index/snii/format/core_metadata.h b/be/src/storage/index/snii/format/core_metadata.h index 621159a803257b..47edf1f93ca092 100644 --- a/be/src/storage/index/snii/format/core_metadata.h +++ b/be/src/storage/index/snii/format/core_metadata.h @@ -17,9 +17,13 @@ #pragma once +#include #include +#include +#include #include "common/status.h" +#include "storage/index/inverted/gram/gram_scheme.h" #include "storage/index/snii/common/slice.h" #include "storage/index/snii/encoding/byte_sink.h" #include "storage/index/snii/format/format_constants.h" @@ -40,10 +44,52 @@ struct SectionRefs { RegionRef bsbf; }; +// Digest of the highest-df terms of one index: (hash, df) pairs small enough to travel with +// the core metadata and stay resident for the life of the segment. +// +// It answers, with no IO at all, the only question the query-side cost gate asks: is this +// node's candidate set already too large for the index to be worth reading? The gate needs +// min(df) across an AND's terms or max(df) across an OR's, and reading those from the +// dictionary costs one remote round trip per term -- measured at 40 seconds on a segment +// whose cache holds nothing, paid in full before the gate decides to give up. +// +// A miss carries as much information as a hit. The digest holds the top-K terms by df, so a +// term absent from it has df <= df_ceiling, and that upper bound is enough for an AND to +// decide. Empty means no bound is available and the caller must read df the usual way. +struct HighDfTerms { + std::vector term_hash; // bsbf_hash of each term, ascending + std::vector df; // parallel to term_hash + uint32_t df_ceiling = 0; // every term not listed has df <= this + + bool empty() const { return term_hash.empty(); } + + // df of `hash` when the digest holds it, else df_ceiling -- an upper bound either way. + // The distinction matters only to a caller that wants to know it was exact. + uint32_t df_upper_bound(uint64_t hash, bool* exact = nullptr) const { + const auto it = std::ranges::lower_bound(term_hash, hash); + if (it != term_hash.end() && *it == hash) { + if (exact != nullptr) { + *exact = true; + } + return df[static_cast(it - term_hash.begin())]; + } + if (exact != nullptr) { + *exact = false; + } + return df_ceiling; + } +}; + struct CoreMetadata { IndexConfig index_config = IndexConfig::kDocsOnly; StatsBlock stats; SectionRefs section_refs; + HighDfTerms high_df_terms; + // The chunking scheme of a gram-family index, recorded by the writer that built it. The + // query side compiles a pattern against the scheme read back from here, so the split a + // pattern is derived with is always the split the data was written with. nullopt on every + // index that is not gram family, which leaves those segments' encoded bytes unchanged. + std::optional gram_scheme; }; Status encode_core_metadata(const CoreMetadata& metadata, ByteSink* out); diff --git a/be/src/storage/index/snii/format/dict_entry.cpp b/be/src/storage/index/snii/format/dict_entry.cpp index 4554cbe560acc9..7a4b912bbfa147 100644 --- a/be/src/storage/index/snii/format/dict_entry.cpp +++ b/be/src/storage/index/snii/format/dict_entry.cpp @@ -38,6 +38,11 @@ std::atomic& body_decode_atomic() { // chain. uint8_t pack_flags(const DictEntry& e) { uint8_t f = 0; + // A dropped posting has no locator, so kind/enc/has_sb describe nothing and are left + // clear rather than carrying whatever the caller happened to leave in the struct. + if (e.posting_dropped) { + return dict_flags::kPostingDropped; + } if (e.kind == DictEntryKind::kInline) { f |= dict_flags::kKind; } @@ -52,6 +57,7 @@ uint8_t pack_flags(const DictEntry& e) { } void apply_flags(uint8_t f, DictEntry* e) { + e->posting_dropped = (f & dict_flags::kPostingDropped) != 0; e->kind = (f & dict_flags::kKind) ? DictEntryKind::kInline : DictEntryKind::kPodRef; e->enc = (f & dict_flags::kEnc) ? DictEntryEnc::kWindowed : DictEntryEnc::kSlim; e->has_sb = (f & dict_flags::kHasSb) != 0; @@ -133,6 +139,11 @@ void write_body(const DictEntry& e, std::string_view prev, IndexTier tier, ByteS write_term_key(e, prev, sink); sink->put_u8(pack_flags(e)); sink->put_varint32(e.df); + // A dropped posting ends the body after df: there is no locator to write, and that + // omission is the saving the flag exists for. + if (e.posting_dropped) { + return; + } if (e.kind == DictEntryKind::kInline) { write_inline(e, tier, sink); } else { @@ -292,7 +303,11 @@ Status decode_dict_entry_rest(ByteSource* src, IndexTier tier, size_t body_start RETURN_IF_ERROR(src->get_u8(&flags)); apply_flags(flags, out); RETURN_IF_ERROR(src->get_varint32(&out->df)); - RETURN_IF_ERROR(read_locator(src, tier, out)); + // A dropped posting wrote no locator, so there is nothing left to read; the body + // length check below still holds it to the bytes the writer produced. + if (!out->posting_dropped) { + RETURN_IF_ERROR(read_locator(src, tier, out)); + } // The body must consume exactly entry_len bytes; otherwise the structure is // inconsistent with the tier. diff --git a/be/src/storage/index/snii/format/dict_entry.h b/be/src/storage/index/snii/format/dict_entry.h index e5060b4c04867a..85ce109ae9408e 100644 --- a/be/src/storage/index/snii/format/dict_entry.h +++ b/be/src/storage/index/snii/format/dict_entry.h @@ -40,7 +40,12 @@ // suffix_len varint # number of suffix bytes // suffix u8[] # suffix bytes that differ from prev_term // flags u8 # bit0 kind / bit1 enc / bit2 has_sb / bit3 -// has_champion(=0) / bit4 offsets_ref(=0) df varint locator: +// has_champion(=0) / bit4 offsets_ref(=0) / bit5 posting_dropped +// df varint +// # bit5 posting_dropped ends the body here: no locator, no payload. bit0-2 are +// # then meaningless. The term is present with a real df but matches every +// # document; it is not the same as a term absent from the dictionary. +// locator: // pod_ref: frq_off_delta varint, frq_len varint, // [prelude_len varint when enc=windowed] // [slim region meta when enc=slim]: @@ -69,6 +74,13 @@ struct DictEntry { DictEntryKind kind = DictEntryKind::kPodRef; DictEntryEnc enc = DictEntryEnc::kSlim; bool has_sb = false; + // stop-gram: the writer dropped this term's posting list. The entry still carries the + // term key and df; kind/enc/has_sb and every locator field below are meaningless and + // nothing after the term stats is encoded. A reader must treat the term as matching + // every document, and must not read a posting for it -- there is none. Keeping the + // entry rather than removing the term is the whole point: an absent term means the + // opposite (matching no document), and conflating the two drops matching rows. + bool posting_dropped = false; // document frequency. uint32_t df = 0; diff --git a/be/src/storage/index/snii/format/format_constants.h b/be/src/storage/index/snii/format/format_constants.h index 3073a18930e009..327ab990ecff9e 100644 --- a/be/src/storage/index/snii/format/format_constants.h +++ b/be/src/storage/index/snii/format/format_constants.h @@ -93,7 +93,14 @@ inline constexpr uint8_t kEnc = 1u << 1; // 0=slim / 1=windowed inline constexpr uint8_t kHasSb = 1u << 2; // posting prelude includes sub-block directory inline constexpr uint8_t kHasChampion = 1u << 3; // v1 always 0 inline constexpr uint8_t kOffsetsRef = 1u << 4; // v1 always 0 -// bit5-7 reserved +// The entry keeps its term key and df but carries no posting locator and no payload: the +// writer dropped a posting list too large for any reader to want. bit0/bit1/bit2 are +// meaningless when this is set, and nothing follows the term stats. A reader that finds +// this entry must treat the term as matching every document -- which is what the query +// side's cost gate already does for a df this high -- and must never confuse it with a +// term absent from the dictionary, which means the opposite: matching no document. +inline constexpr uint8_t kPostingDropped = 1u << 5; +// bit6-7 reserved } // namespace dict_flags enum class DictEntryKind : uint8_t { kPodRef = 0, kInline = 1 }; @@ -113,6 +120,33 @@ enum class PrxCodec : uint8_t { kPfor = 2 /* bit7 cont-reserved */ }; +// ---- High-df term digest (SniiHighDfTermsPB) ---- +// +// Which terms the writer records so a query can bound its candidate count without reading +// the dictionary. Both numbers are properties of the format rather than of any dataset: the +// first says how rare a term has to be before bounding it stops being useful, the second +// caps what the digest costs to carry. +// +// A term enters the digest when its df exceeds doc_count / kHighDfDigestDivisor. The divisor +// sits an order of magnitude below the candidate ratios a gate ever accepts, so the digest +// covers every term a gate could reject and then some -- a term below the floor is one no +// gate would have given up on anyway. +inline constexpr uint64_t kHighDfDigestDivisor = 2000; // floor = doc_count / 2000 (5 bp) +// Absolute cap on entries. At ~11 bytes each this bounds the digest at ~45 KB, one request +// alongside the core metadata the reader already fetches. Real log text puts 88.6% of all +// posting entries in the top 0.7% of terms, so this binds only on unusually flat +// vocabularies -- and when it does, the digest still yields a valid (looser) bound. +inline constexpr size_t kMaxHighDfDigestTerms = 4096; +// Proportional cap: at most one entry per this many distinct terms. The absolute cap alone +// is not enough on a small segment, where the df floor falls to a couple of documents, most +// of the vocabulary clears it, and the digest would fill with terms that are "common" only +// in the sense of occurring twice -- 45 KB of digest on a 96 KB index. On a large segment +// the floor binds first and this never engages. Measured overhead with both caps in force: +// 0.30%-0.36% of the index across the density x max_gram grid. +inline constexpr uint64_t kHighDfDigestVocabularyShare = 64; +// Floor on the proportional cap, so a tiny vocabulary still gets a usable digest. +inline constexpr size_t kMinHighDfDigestTerms = 64; + // ---- Build-time parameters (not format semantics; may be tuned against real // metrics) ---- inline constexpr uint32_t kFrqBaseUnit = 256; // window base unit diff --git a/be/src/storage/index/snii/query/docid_conjunction.cpp b/be/src/storage/index/snii/query/docid_conjunction.cpp index df998d80dbc267..3aab9d9103e98a 100644 --- a/be/src/storage/index/snii/query/docid_conjunction.cpp +++ b/be/src/storage/index/snii/query/docid_conjunction.cpp @@ -68,6 +68,21 @@ Status posting_abs_offset(const LogicalIndexReader& idx, uint64_t base, uint64_t Status configure_term_plan(const LogicalIndexReader& idx, bool need_positions, io::BatchRangeFetcher* fetcher, TermPlan* p) { + // A stop-gram term has no posting list to plan: the writer dropped it because the + // query-side cost gate would have refused to read it anyway. Reaching here means the + // caller intends to consume the postings as ground truth, and this term cannot supply + // them -- so refuse the index rather than answer from a term that matches everything. + // Callers whose semantics survive a match-all term (the gram boolean query, whose + // candidates are always re-checked against the column) drop such terms before + // planning and never arrive here. This is the single choke point every posting read + // passes through, which makes the failure mode of forgetting that "skip the index", + // not "return rows that do not match". + if (p->entry.posting_dropped) { + return Status::Error( + "docid_conjunction: term '{}' has a dropped posting list (stop-gram); this " + "query needs exact postings", + p->entry.term); + } p->df = p->entry.df; p->pod_ref = (p->entry.kind == DictEntryKind::kPodRef); p->windowed = p->pod_ref && p->entry.enc == DictEntryEnc::kWindowed; diff --git a/be/src/storage/index/snii/query/gram_boolean_query.cpp b/be/src/storage/index/snii/query/gram_boolean_query.cpp new file mode 100644 index 00000000000000..4b62b782ec01c6 --- /dev/null +++ b/be/src/storage/index/snii/query/gram_boolean_query.cpp @@ -0,0 +1,552 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/gram_boolean_query.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "storage/index/snii/format/bsbf.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_union.h" + +namespace doris::snii::query { + +// R8/R24 (unity build): file-level helper implementations live in this named namespace private +// to the file rather than in a bare anonymous one, so they cannot clash with symbols of other +// files under a unity build. +namespace gram_boolean_query_detail { + +// Behaves exactly like RoaringDocIdSink in the anonymous namespace of snii_index_reader.cpp (a +// non-empty batch goes through addMany, a non-empty range through addRange(first, +// last_exclusive), and dedups()==true lets a multi-gram OR/AND stream postings straight into the +// same bitmap); that class cannot be reused across translation units, hence this copy. +class RoaringSink final : public DocIdSink { +public: + explicit RoaringSink(roaring::Roaring* bitmap) : _bitmap(bitmap) {} + + Status append_sorted(std::span docids) override { + if (!docids.empty()) { + _bitmap->addMany(docids.size(), docids.data()); + } + return Status::OK(); + } + + Status append_range(uint32_t first, uint64_t last_exclusive) override { + if (last_exclusive > first) { + _bitmap->addRange(first, last_exclusive); + } + return Status::OK(); + } + + bool dedups() const override { return true; } + +private: + roaring::Roaring* _bitmap; +}; + +// The cost gate of gram_boolean_query(): true when a candidate set of `candidates` rows is too +// large a fraction of the segment for the index IO to pay for itself. It is a COST predicate +// only -- the caller answers true by widening the node to the whole docid space, never by +// dropping rows. config::gram_index_max_candidate_ratio_bp == 0 disables the gate; a value of +// 10000 or more can never fire because a candidate set never exceeds the segment. +// True when `candidates` is too large for the index to pay for itself on this segment. +// +// `budget` comes from the segment's own term distribution (GramPostingSource:: +// candidate_budget) and is preferred when available: it is the only form of this predicate +// that does not need retuning when the dataset changes. The basis-point ratio below is the +// fallback for segments written before the digest existed, and stays available as an +// override for anyone who needs to pin the old behaviour exactly. +bool exceeds_candidate_budget(uint64_t candidates, uint32_t num_docs, uint64_t budget) { + if (budget > 0) { + return candidates > budget; + } + const int32_t bp = config::gram_index_max_candidate_ratio_bp; + if (bp <= 0 || num_docs == 0) { + return false; + } + // Row floor. Below it the segment's whole gram index is a few KB and one or two requests: + // giving up saves nothing measurable, and costs the pruning the index really does deliver at + // that size. The ratio only starts to mean something once the skipped index IO can outweigh + // the rows it stops eliminating. + const int32_t min_rows = config::gram_index_candidate_ratio_min_rows; + if (min_rows > 0 && num_docs < static_cast(min_rows)) { + return false; + } + return candidates * 10000 > static_cast(num_docs) * static_cast(bp); +} + +// How many documents an AND over gram postings with these dfs is likely to keep. +// +// The gate used min(df), which is the exact upper bound: an intersection can never hold more +// docs than its rarest member. For grams cut out of a single ASCII literal that bound is also +// close to the truth, because those grams overlap almost perfectly -- they were produced by +// sliding over the same bytes, so a row holding one very likely holds the rest, and the +// intersection really is about as big as the rarest of them. +// +// Nothing guarantees that overlap, and one column shape breaks it completely. A CJK column +// indexes one term per code point, so the grams of a four-character literal are four ordinary +// characters that share little beyond the rows actually spelling it. Measured on weibo, a +// literal whose rarest character was still a common one had five matching rows and was +// abandoned by the gate, having eliminated none: min(df) said the node was a fifth of the +// segment when it was five documents. +// +// The opposite extreme, multiplying the selectivities as if the grams were independent, +// understates a correlated node by just as much and would disable the gate on the ASCII +// patterns it was built for. This takes the standard middle course: sort by df, take the +// rarest gram as the base, and let every further gram contribute its selectivity under an +// exponentially decaying exponent -- the second counts most, the third half as much, and so +// on. The result is never above min(df) and never below the independence product, so it +// degrades to the old behaviour exactly where the old behaviour was right, and it needs no +// statistics beyond the dfs the caller has already read. +// +// This is an estimate and not a bound, which the gate can afford: giving up widens a node to +// the whole segment and proceeding reads postings, so being wrong in either direction costs +// IO and never a row. +uint64_t estimate_and_candidates(std::vector dfs, uint32_t num_docs) { + if (dfs.empty() || num_docs == 0) { + return 0; + } + std::ranges::sort(dfs); + if (dfs.front() == 0) { + return 0; + } + double estimate = static_cast(dfs.front()); + double exponent = 1.0; + for (size_t i = 1; i < dfs.size(); ++i) { + exponent *= 0.5; + if (dfs[i] == 0) { + return 0; + } + const double selectivity = + std::min(1.0, static_cast(dfs[i]) / static_cast(num_docs)); + estimate *= std::pow(selectivity, exponent); + } + // Round up so a node that survives at all is never estimated at zero documents, and clamp + // to the exact upper bound: floating point must not let the estimate drift above min(df). + const double rounded = std::ceil(estimate); + return std::min(static_cast(rounded), dfs.front()); +} + +Status eval(GramPostingSource& src, const segment_v2::gram::GramQuery& q, uint32_t num_docs, + roaring::Roaring* out, GramGateStats* gate_stats); + +// Records one node the gate widened rather than read. Only cost decisions pass through here: +// widening because a node means ALL (an AND with no grams, an OR holding a stop-gram) is not a +// gate decision and must stay uncounted, or the counter stops meaning "the index gave up". +void note_gave_up(GramGateStats* gate_stats) { + if (gate_stats != nullptr) { + ++gate_stats->nodes_given_up; + } +} + +// AND. The order here is deliberate and each step earns its place: +// +// 1. one batched dictionary lookup for the gram leaves. A gram missing from the segment makes +// the whole node definitively empty, which is where most of this index's value comes from, +// and finding that out costs one dictionary round and no postings. +// 2. the sub-queries, evaluated before the leaves are read. An AND is commutative, so this +// changes no answer, and it means the sub-query's cardinality -- an exact upper bound on +// the whole node -- is in hand when the gate judges the leaves. Judging the leaves alone +// is how `(A|B)C` was abandoned: the grams of C were common, and the alternation that held +// the node to a handful of rows was never looked at. +// 3. the cost gate, on the smaller of the leaf estimate and what the sub-queries already +// narrowed to. +// 4. the leaf postings, through one batched conjunction. +// +// An AND with neither gram leaves nor sub-queries counts as ALL. +Status eval_and(GramPostingSource& src, const segment_v2::gram::GramQuery& q, uint32_t num_docs, + roaring::Roaring* out, GramGateStats* gate_stats) { + bool has_acc = false; + roaring::Roaring acc; + std::vector constraining_grams; + std::vector constraining_dfs; + const uint64_t budget = src.candidate_budget(); + + if (!q.grams.empty()) { + // Cheapest arm of the gate, and the only one that costs nothing. The estimate needs + // every gram's df, so it takes an exactly-known df for EVERY gram: the digest holds + // only the segment's common terms, and a gram it cannot bound is one rarer than the + // digest's floor -- exactly the gram most likely to carry the node. Estimating + // without it would systematically overstate the node and give up on the patterns the + // index exists for. When they are all known no dictionary block is read at all, which + // on a cold segment is the difference between 40 seconds and none. + // + // It is skipped entirely when the node has sub-queries: nothing here can bound those, + // and giving up on the leaves while a sub-query holds the node to a few rows is the + // exact mistake this function is arranged to avoid. Such a node pays one dictionary + // round instead. + if (budget > 0 && q.subs.empty()) { + std::vector known_df; + std::vector known; + src.known_dfs(q.grams, &known_df, &known); + bool all_known = true; + for (size_t i = 0; i < q.grams.size(); ++i) { + if (known[i] == 0) { + all_known = false; + break; + } + } + if (all_known && exceeds_candidate_budget(estimate_and_candidates(known_df, num_docs), + num_docs, budget)) { + note_gave_up(gate_stats); + out->addRange(0, num_docs); + return Status::OK(); + } + } + std::vector dfs; + RETURN_IF_ERROR(src.dfs(q.grams, &dfs)); + for (const GramDf& gram_df : dfs) { + if (!gram_df.found) { + // The index only produces a superset of candidates: a missing gram makes the + // whole AND branch definitively empty, so return right away without reading any + // posting -- and without evaluating a sub-query that could not change it. + return Status::OK(); + } + } + // A stop-gram matches every document, so it narrows nothing and has no posting to + // read. Dropping it from the conjunction leaves the node's meaning unchanged + // (X AND ALL == X) and keeps it away from the planner, which refuses such a term + // outright. When they are all stop-grams the gram leaves constrain nothing at all: + // the node becomes its sub-queries, or ALL if it has none, which is exactly what the + // tail below already does. + constraining_grams.reserve(q.grams.size()); + constraining_dfs.reserve(dfs.size()); + for (size_t i = 0; i < q.grams.size(); ++i) { + if (!dfs[i].matches_all) { + constraining_grams.push_back(q.grams[i]); + constraining_dfs.push_back(dfs[i]); + } + } + } + + for (const auto& sub : q.subs) { + roaring::Roaring cur; + RETURN_IF_ERROR(eval(src, sub, num_docs, &cur, gate_stats)); + if (!has_acc) { + acc = std::move(cur); + has_acc = true; + } else { + acc &= cur; + } + if (acc.isEmpty()) { + return Status::OK(); + } + } + + if (!constraining_grams.empty()) { + // Cost gate, before any posting read. Giving up here rather than after and_postings() + // is the entire point: a node that turns out too large has already paid for every + // posting it read by then, and measurements on remote object storage show that cost is + // not recovered -- the surviving candidates are too scattered for a single page of the + // column to be skipped, so the column is read in full anyway and the index reads are + // pure overhead, up to a 1.4x regression against not using the index at all. + // + // The node is bounded by two things: what its own leaves are likely to intersect to, + // and what the sub-queries have already narrowed it to. The second is not an estimate + // at all -- those rows have been counted -- so it wins whenever it is smaller. + uint64_t estimate = estimate_and_candidates( + [&] { + std::vector values; + values.reserve(constraining_dfs.size()); + for (const GramDf& gram_df : constraining_dfs) { + values.push_back(gram_df.df); + } + return values; + }(), + num_docs); + if (has_acc) { + estimate = std::min(estimate, acc.cardinality()); + } + if (exceeds_candidate_budget(estimate, num_docs, budget)) { + note_gave_up(gate_stats); + out->addRange(0, num_docs); + return Status::OK(); + } + roaring::Roaring leaves; + RETURN_IF_ERROR(src.and_postings(constraining_grams, &leaves)); + if (!has_acc) { + acc = std::move(leaves); + has_acc = true; + } else { + acc &= leaves; + } + if (acc.isEmpty()) { + return Status::OK(); + } + } + + if (!has_acc) { + // Neither gram leaves nor sub-queries: the AND degenerates to ALL. + out->addRange(0, num_docs); + return Status::OK(); + } + *out |= acc; + return Status::OK(); +} + +// OR: union the postings of all gram leaves held directly through one batched union, then union +// the evaluation result of every sub-query. +Status eval_or(GramPostingSource& src, const segment_v2::gram::GramQuery& q, uint32_t num_docs, + roaring::Roaring* out, GramGateStats* gate_stats) { + if (!q.grams.empty()) { + const uint64_t budget = src.candidate_budget(); + // The mirror of the AND's no-IO arm, and a cheaper test: a union is at least as large + // as any one member, so a SINGLE gram known to be over budget settles the node. No + // need for the others to be known at all. + if (budget > 0) { + std::vector known_df; + std::vector known; + src.known_dfs(q.grams, &known_df, &known); + for (size_t i = 0; i < q.grams.size(); ++i) { + if (known[i] != 0 && exceeds_candidate_budget(known_df[i], num_docs, budget)) { + note_gave_up(gate_stats); + out->addRange(0, num_docs); + return Status::OK(); + } + } + } + std::vector dfs; + RETURN_IF_ERROR(src.dfs(q.grams, &dfs)); + // A union is at least as large as its largest member, so max(df) over the gram leaves is + // a lower bound on this node's candidate count -- enough to fire the cost gate before a + // single posting (or any sub-query) is read. + uint64_t max_df = 0; + for (const GramDf& gram_df : dfs) { + if (gram_df.matches_all) { + // One member that matches every document makes the whole union everything, + // whatever the others hold and whatever the sub-queries add. Nothing left to + // read. + out->addRange(0, num_docs); + return Status::OK(); + } + if (gram_df.found) { + max_df = std::max(max_df, gram_df.df); + } + } + if (exceeds_candidate_budget(max_df, num_docs, budget)) { + note_gave_up(gate_stats); + out->addRange(0, num_docs); + return Status::OK(); + } + RETURN_IF_ERROR(src.or_postings(q.grams, out)); + } + for (const auto& sub : q.subs) { + roaring::Roaring cur; + RETURN_IF_ERROR(eval(src, sub, num_docs, &cur, gate_stats)); + *out |= cur; + } + return Status::OK(); +} + +Status eval(GramPostingSource& src, const segment_v2::gram::GramQuery& q, uint32_t num_docs, + roaring::Roaring* out, GramGateStats* gate_stats) { + using Op = segment_v2::gram::GramQuery::Op; + switch (q.op) { + case Op::ALL: + out->addRange(0, num_docs); + return Status::OK(); + case Op::NONE: + return Status::OK(); + case Op::AND: + return eval_and(src, q, num_docs, out, gate_stats); + case Op::OR: + return eval_or(src, q, num_docs, out, gate_stats); + } + return Status::OK(); +} + +} // namespace gram_boolean_query_detail + +Status LogicalIndexPostingSource::_resolve(const std::vector& grams) { + std::vector missing; + missing.reserve(grams.size()); + for (const std::string& gram : grams) { + if (!_resolved.contains(gram)) { + missing.push_back(gram); + } + } + if (missing.empty()) { + return Status::OK(); + } + // lookup_batch() requires a sorted, duplicate-free batch and returns results aligned with it; + // the memo below maps them back to the caller's order. + std::sort(missing.begin(), missing.end()); + missing.erase(std::unique(missing.begin(), missing.end()), missing.end()); + std::vector results; + RETURN_IF_ERROR(_idx.lookup_batch(missing, &results)); + for (size_t i = 0; i < missing.size(); ++i) { + _resolved.emplace(std::move(missing[i]), std::move(results[i])); + } + return Status::OK(); +} + +// The gate's budget, taken from this segment's own term distribution instead of a ratio +// configured for some other dataset. +// +// The digest holds the segment's highest-df terms and reports the ceiling below which every +// term it omitted sits. That ceiling is the segment's own answer to "how common is common +// here", which is exactly the question a fixed basis-point ratio has to guess at -- and it +// guesses badly, because it applies one proportion to a 500k-row table and a 34M-row one +// whose page counts differ seventyfold. +// +// The multiplier says how far above that line a node has to be before pruning stops paying. +// It is dimensionless and stable across datasets, which is what a configured ratio is not: +// on a flat vocabulary the digest fills to its cap, the ceiling rises, and the budget rises +// with it -- the segment widens its own gate exactly when its grams are least selective. +// Small segments move the other way: their ceiling falls toward zero, the gate tightens, and +// the index stops being read for a table a scan handles trivially. That is the behaviour the +// configured row floor used to approximate. +uint64_t LogicalIndexPostingSource::candidate_budget() const { + const auto& digest = _idx.high_df_terms(); + if (digest.empty() && digest.df_ceiling == 0) { + // Written before the digest existed: no basis, caller falls back to the config. + return 0; + } + return static_cast(digest.df_ceiling) * kCandidateBudgetCeilingMultiple; +} + +void LogicalIndexPostingSource::known_dfs(const std::vector& grams, + std::vector* out, + std::vector* known) const { + out->assign(grams.size(), 0); + known->assign(grams.size(), 0); + const auto& digest = _idx.high_df_terms(); + if (digest.empty()) { + return; + } + for (size_t i = 0; i < grams.size(); ++i) { + bool exact = false; + const uint32_t df = digest.df_upper_bound(format::bsbf_hash(grams[i]), &exact); + if (exact) { + (*out)[i] = df; + (*known)[i] = 1; + } + } +} + +Status LogicalIndexPostingSource::dfs(const std::vector& grams, + std::vector* out) { + out->assign(grams.size(), GramDf {}); + if (grams.empty()) { + return Status::OK(); + } + RETURN_IF_ERROR(_resolve(grams)); + for (size_t i = 0; i < grams.size(); ++i) { + const auto it = _resolved.find(grams[i]); + if (it == _resolved.end() || !it->second.found) { + continue; + } + (*out)[i].found = true; + (*out)[i].df = it->second.entry.df; + (*out)[i].matches_all = it->second.entry.posting_dropped; + } + return Status::OK(); +} + +Status LogicalIndexPostingSource::and_postings(const std::vector& grams, + roaring::Roaring* out) { + if (grams.empty()) { + return Status::OK(); + } + RETURN_IF_ERROR(_resolve(grams)); + std::vector resolved; + resolved.reserve(grams.size()); + for (const std::string& gram : grams) { + const auto it = _resolved.find(gram); + if (it == _resolved.end() || !it->second.found) { + // A gram missing from the dictionary makes the intersection definitively empty. + return Status::OK(); + } + resolved.push_back({it->second.entry, it->second.frq_base, it->second.prx_base}); + } + // The same plan/fetch/leapfrog pipeline boolean_and() runs, minus the per-term dictionary + // resolution it would repeat: one batched prelude round, then a docid-only conjunction that + // drives on the rarest gram and skips whole windows of the denser ones. + io::BatchRangeFetcher round1(_idx.reader()); + std::vector plans; + RETURN_IF_ERROR(internal::plan_resolved_terms(_idx, std::move(resolved), &round1, &plans, + /*need_positions=*/false)); + if (round1.pending() > 0) { + RETURN_IF_ERROR(round1.fetch()); + } + RETURN_IF_ERROR(internal::open_preludes(round1, &plans, /*need_positions=*/false)); + std::vector docids; + RETURN_IF_ERROR(internal::build_docid_only_conjunction(_idx, round1, plans, &docids)); + if (!docids.empty()) { + out->addMany(docids.size(), docids.data()); + } + return Status::OK(); +} + +Status LogicalIndexPostingSource::or_postings(const std::vector& grams, + roaring::Roaring* out) { + if (grams.empty()) { + return Status::OK(); + } + RETURN_IF_ERROR(_resolve(grams)); + std::vector postings; + postings.reserve(grams.size()); + for (const std::string& gram : grams) { + const auto it = _resolved.find(gram); + if (it == _resolved.end() || !it->second.found) { + continue; + } + postings.push_back({it->second.entry, it->second.frq_base, it->second.prx_base}); + } + if (postings.empty()) { + return Status::OK(); + } + // Same union pipeline boolean_or() runs: windowed postings share one prelude round and one + // docid round, and each posting is streamed straight into the (deduplicating) bitmap. + gram_boolean_query_detail::RoaringSink sink(out); + return internal::emit_docid_union(_idx, postings, &sink); +} + +Status gram_boolean_query(GramPostingSource& src, const segment_v2::gram::GramQuery& q, + uint32_t num_docs, roaring::Roaring* out, GramGateStats* gate_stats) { + RETURN_IF_ERROR(gram_boolean_query_detail::eval(src, q, num_docs, out, gate_stats)); + // Second arm of the cost gate, for the shapes whose size no df bound could predict (a + // multi-gram AND above all). It cannot give the IO back, but it avoids an equally pointless + // bitmap intersection above and makes "this index pruned nothing" visible instead of hiding + // it behind a candidate set that happens to be nearly the whole segment. Widening to the full + // range only ever adds rows. + const uint64_t candidates = out->cardinality(); + if (gram_boolean_query_detail::exceeds_candidate_budget(candidates, num_docs, + src.candidate_budget())) { + // Only a result that still holds something back is a give-up here. A node that already + // abandoned itself above arrives as the whole segment and would otherwise be counted + // twice, which turns the counter from "nodes the gate widened" into a number no reader + // can interpret. + if (candidates < num_docs) { + gram_boolean_query_detail::note_gave_up(gate_stats); + } + out->addRange(0, num_docs); + } + return Status::OK(); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/gram_boolean_query.h b/be/src/storage/index/snii/query/gram_boolean_query.h new file mode 100644 index 00000000000000..9bf25f3ae26d2b --- /dev/null +++ b/be/src/storage/index/snii/query/gram_boolean_query.h @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/gram/gram_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// gram_boolean_query -- evaluates a gram::GramQuery boolean query tree against the gram-family +// dictionary/postings of one SNII segment, producing a docid bitmap. The index may only narrow +// the candidate set: a missing gram, an unsupported query shape or a failed lookup can only make +// the layer above degrade to "no acceleration" and must never change the query result, which is +// why every code path here returns a Status instead of asserting that a gram must exist. +namespace doris::snii::query { + +// One gram's dictionary resolution, aligned position by position with the gram vector handed to +// GramPostingSource::dfs(). found=false means the gram is absent from this segment's dictionary. +// How far above the segment's own "commonness line" a node has to sit before pruning stops +// paying. The line itself comes from the segment (the high-df digest's ceiling); this is the +// dimensionless factor applied to it, and unlike a basis-point ratio it does not have to be +// retuned per dataset -- it scales with whatever that segment's distribution turns out to be. +// +// 3 reproduces the behaviour the previously configured 15 bp had on the measured corpora: on +// an untruncated digest the ceiling sits at one part in kHighDfDigestDivisor (5 bp), and +// three times that is the same line the sweep found admitted no regression. +inline constexpr uint64_t kCandidateBudgetCeilingMultiple = 3; + +struct GramDf { + bool found = false; + uint64_t df = 0; + // stop-gram: the gram is in the dictionary with a real df, but its posting list was + // dropped at write time, so it constrains nothing -- it matches every document. It is + // the opposite of !found, which matches none, and the two must never be conflated: one + // widens a node, the other empties it. + bool matches_all = false; +}; + +// The posting data source gram_boolean_query() consumes. Every entry point takes the WHOLE gram +// list of one query node rather than a single gram: on object storage each dictionary lookup and +// each posting read is an independent remote request, so a per-gram interface turns a k-gram node +// into O(k) serial round trips whose latency dominates the query. In production this is +// LogicalIndexPostingSource below, adapting a LogicalIndexReader; tests inject a map-based fake so +// that the AND/OR/ALL/NONE evaluation logic can be covered without building real index files. +class GramPostingSource { +public: + virtual ~GramPostingSource() = default; + + // Look up the document frequency (df) of every gram in one batch. out is aligned with grams; + // out[i].found=false means that gram is not in the dictionary, in which case every AND node + // containing it evaluates to NONE (the empty set) and no posting list is read at all. + // out[i].matches_all means the opposite: the gram IS in the dictionary but its posting list + // was dropped at write time (stop-gram), so it matches every document and constrains nothing. + virtual Status dfs(const std::vector& grams, std::vector* out) = 0; + + // Decode the intersection of the docid sets of all grams (without positions or term + // frequencies) into out. Only called after dfs() has confirmed that every gram exists, and + // never with a gram dfs() reported as matches_all -- the evaluator drops those first, since + // they have no posting list to intersect and would not narrow the result if they had one. + virtual Status and_postings(const std::vector& grams, roaring::Roaring* out) = 0; + + // Decode the union of the docid sets of all grams into out. Grams absent from the dictionary + // simply contribute nothing to the union. Never called when any gram is matches_all: such a + // union is the whole segment, which the evaluator answers from the df batch alone. + virtual Status or_postings(const std::vector& grams, roaring::Roaring* out) = 0; + + // ---- The no-IO arm of the cost gate ---- + // + // dfs() above costs one remote dictionary round trip per gram -- 40 seconds on a segment + // whose cache holds nothing, and the gate frequently spends it only to conclude the node + // is not worth pruning. These two calls let it reach that conclusion first, for free. + + // The candidate count above which this segment is not worth pruning, derived from the + // segment's own term distribution. 0 means the source cannot supply one and the caller + // falls back to the configured ratio. + virtual uint64_t candidate_budget() const { return 0; } + + // Exact df for the grams the source can answer without IO; known[i] == 0 for the rest, + // whose df is then only bounded from above (by the budget's own basis) and so might + // still be rare enough to be worth reading. out and known are sized to grams. + // + // The asymmetry is the point. An AND is estimated from ALL its dfs, so one unknown gram + // sinks the arm: the digest holds only common terms, so an unknown gram is a rare one -- + // exactly the gram most likely to carry the node. An OR is the mirror, because a union is + // at least as large as any member: one known gram over budget settles it. + virtual void known_dfs(const std::vector& grams, std::vector* out, + std::vector* known) const { + out->assign(grams.size(), 0); + known->assign(grams.size(), 0); + } +}; + +// The production GramPostingSource on top of LogicalIndexReader. All three entry points share one +// request-scoped dictionary memo: dfs() resolves a batch through LogicalIndexReader::lookup_batch +// (bounded waves of concurrent DICT block reads instead of one blocking read per gram) and keeps +// the resolved DictEntry, so the and_postings()/or_postings() call that follows reads postings +// straight from those entries and never looks the same gram up a second time. Posting reads then +// go through the same batched plan/fetch pipeline boolean_and()/boolean_or() use, so a node costs +// a constant number of round trips instead of one per gram. The memo lives exactly as long as +// this object, which production creates per index-query call. +class LogicalIndexPostingSource final : public GramPostingSource { +public: + explicit LogicalIndexPostingSource(const reader::LogicalIndexReader& idx) : _idx(idx) {} + Status dfs(const std::vector& grams, std::vector* out) override; + Status and_postings(const std::vector& grams, roaring::Roaring* out) override; + Status or_postings(const std::vector& grams, roaring::Roaring* out) override; + uint64_t candidate_budget() const override; + void known_dfs(const std::vector& grams, std::vector* out, + std::vector* known) const override; + +private: + // Resolves, in one batch, every gram of the list that the memo does not hold yet. + Status _resolve(const std::vector& grams); + + const reader::LogicalIndexReader& _idx; + std::unordered_map _resolved; +}; + +// Evaluate q against src: ALL -> [0, num_docs); NONE -> the empty set; AND looks up the df of all +// of its gram leaves in one batch (a single missing leaf short-circuits the whole node to empty +// without reading any posting), intersects the leaves through one batched conjunction, then +// intersects with each sub-query and returns early as soon as the intersection is empty (an AND +// with neither leaves nor sub-queries degenerates to ALL); OR unions the postings of all its gram +// leaves through one batched union, then unions the result of every sub-query. The recursion +// depth is bounded by the query tree produced by RegexGramCompiler's depth-limited analysis. +// +// COST GATE (config::gram_index_max_candidate_ratio_bp, in basis points of the segment; 0 +// disables it; applied only to segments of at least config::gram_index_candidate_ratio_min_rows +// rows). When the candidate set +// is a large fraction of a segment big enough for that to matter, the index IO is paid in full +// for almost no pruning, so a node whose estimated candidate count already exceeds the ratio +// gives up before reading any posting -- an AND is estimated from all its dfs together, an OR +// from its largest, which is an exact lower bound on a union -- and the final result gives up if +// it ends up above the ratio anyway. +// "Giving up" is a cost decision and NEVER a semantic one: it yields the whole [0, num_docs) +// range, so the caller's `row_bitmap &= result` is a no-op and not a single row can be pruned +// away. Widening a candidate set is always safe (the expression above re-verifies every candidate +// row); narrowing it is not, which is why no path here may ever produce fewer docids than the +// true candidate set. +// What the gate decided, for the profile. Every defect this gate has had presented the same way +// from outside -- the index was consulted and eliminated nothing -- and a profile could only show +// it indirectly, as a candidate count equal to the segment with zero rows filtered. That is an +// inference a reader has to already suspect to make; this is the decision itself. +struct GramGateStats { + // Nodes the cost gate widened to the whole segment rather than read. Counts gate decisions + // only: an AND with no grams, or an OR holding a stop-gram, is ALL for reasons of meaning + // rather than cost and is not counted here. + uint32_t nodes_given_up = 0; +}; + +Status gram_boolean_query(GramPostingSource& src, const segment_v2::gram::GramQuery& q, + uint32_t num_docs, roaring::Roaring* out, + GramGateStats* gate_stats = nullptr); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/reader/logical_index_reader.cpp b/be/src/storage/index/snii/reader/logical_index_reader.cpp index 5f146397d667ac..5ce803da145bcf 100644 --- a/be/src/storage/index/snii/reader/logical_index_reader.cpp +++ b/be/src/storage/index/snii/reader/logical_index_reader.cpp @@ -1085,6 +1085,16 @@ Status resolve_window(const format::RegionRef& section, uint64_t base, uint64_t Status LogicalIndexReader::resolve_frq_window(const format::DictEntry& entry, uint64_t frq_base, uint64_t* abs_off, uint64_t* len) const { + // Second guard on the stop-gram contract, for the readers that reach a posting without + // going through the query planner (the docid union, scoring, the compaction cursor). + // A dropped entry's locator fields are all zero, so without this the caller would be + // handed a zero-length window at the region base and read it as an empty posting -- + // "matches nothing", the exact inversion of what a dropped term means. Refusing is + // what lets the caller fall back to reading rows. + if (entry.posting_dropped) { + return Status::Error( + "logical_index: term '{}' has a dropped posting list (stop-gram)", entry.term); + } return resolve_window(section_refs().posting_region, frq_base, entry.frq_off_delta, entry.frq_len, entry.prelude_len, abs_off, len); } diff --git a/be/src/storage/index/snii/reader/logical_index_reader.h b/be/src/storage/index/snii/reader/logical_index_reader.h index ee1e90b2571687..31c15c4f2e5ca1 100644 --- a/be/src/storage/index/snii/reader/logical_index_reader.h +++ b/be/src/storage/index/snii/reader/logical_index_reader.h @@ -176,11 +176,23 @@ class LogicalIndexReader { const format::SectionRefs& section_refs() const { return core_.section_refs; } const format::StatsBlock& stats() const { return core_.stats; } + // Bounds on the df of this index's most common terms, resident since the segment was + // opened. Lets a caller decide a term is too common to be worth reading without issuing + // the dictionary read that would tell it exactly how common. Empty on indexes written + // before the digest existed, in which case there is no bound and df must be read. + const format::HighDfTerms& high_df_terms() const { return core_.high_df_terms; } format::IndexTier tier() const { return tier_; } bool has_positions() const { return has_positions_; } // Whether BM25 norms exist: current writers emit them for analyzed indexes with positions. bool has_norms() const { return core_.section_refs.norms.length != 0; } LogicalIndexOpenMode open_mode() const { return open_mode_; } + // The chunking scheme of a gram-family index, read back from the core metadata this segment + // carries. It is what the query side compiles a LIKE / REGEXP pattern against, so a pattern + // is always derived with the very scheme the writer used, whatever the index policy says + // today. nullopt means the index is not gram family and no pattern may be pushed down to it. + const std::optional& gram_scheme() const { + return core_.gram_scheme; + } io::FileReader* reader() const { return reader_; } // Returns a reader over the validated norms section. The first call reads diff --git a/be/src/storage/index/snii/snii_index_writer.cpp b/be/src/storage/index/snii/snii_index_writer.cpp index 5d6d3094266fa6..ce6943112f1e93 100644 --- a/be/src/storage/index/snii/snii_index_writer.cpp +++ b/be/src/storage/index/snii/snii_index_writer.cpp @@ -30,7 +30,9 @@ #include "storage/index/index_file_writer.h" #include "storage/index/inverted/analyzer/analyzer.h" #include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/inverted/gram/gram_density.h" #include "storage/index/inverted/query/query_info.h" +#include "storage/index/inverted/tokenizer/ngram/gram_tokenizer.h" #include "storage/index/snii/query/bm25_scorer.h" #include "storage/index/snii/writer/global_memory_limiter.h" #include "storage/index/snii/writer/snii_build_memory_tracker.h" @@ -45,6 +47,76 @@ SniiIndexColumnWriter::SniiIndexColumnWriter(IndexFileWriter* index_file_writer, _index_meta(index_meta), _is_char(value_type == FieldType::OLAP_FIELD_TYPE_CHAR) {} +// Gram-family detection (Rulings R21/R22): the scheme comes only from the single analyzer +// provider the writer created itself -- the same provider both produces the actual tokenizer and +// answers "am I gram family?", so the two cannot drift. A built-in analyzer +// (standard/english/...) goes through BuiltinAnalyzerProvider and the base class default, which +// is always nullopt, and the policy manager is never consulted (consulting it would throw +// "Policy not found" and make every built-in-analyzer index impossible to build). +void SniiIndexColumnWriter::_apply_gram_family_scheme( + const inverted_index::AnalyzerProviderPtr& analyzer_provider) { + if (analyzer_provider != nullptr) { + _gram_scheme = analyzer_provider->gram_scheme(); + } + // An index-level char_filter is wrapped around the reader by the writer itself + // (create_reader) and is invisible to the provider: once one exists, the stored term is no + // longer equal to GramExtractor.extract(raw column value), breaking the row invariant the + // query side (phase C) relies on, so this is treated as "not gram family" (fail-safe, for + // the same reason as R22). + if (!_analyzer_config.char_filter_map.empty()) { + _gram_scheme.reset(); + } + DCHECK(!_gram_scheme.has_value() || _should_analyzer); + if (!_gram_scheme.has_value() || !_has_positions) { + return; + } + // R15: a gram-family hit forces a degradation to docs-only (the gram index does not support + // phrase positions), and it has to happen before SpimiTermBuffer is fixed by _has_positions. + LOG(INFO) << "gram-family analyzer forces docs-only index, ignoring support_phrase for index " + << _index_meta->index_id(); + _has_positions = false; + _config = ::doris::snii::format::IndexConfig::kDocsOnly; +} + +// Arms the density solve for this segment. The configured density stays as the fallback: it is +// what a segment gets when the feature is off, when the sample carries no window of the +// promised length, or when nothing at all is written. +void SniiIndexColumnWriter::_arm_density_calibration() { + if (!_gram_scheme.has_value() || !config::enable_gram_index_adaptive_density) { + return; + } + const auto min_literal = static_cast(config::gram_index_min_literal_bytes); + _density_solver = std::make_unique(min_literal, _gram_scheme->max_len); + _density_calibrating = true; +} + +// The whole path creates exactly one analyzer provider, and it must come before SpimiTermBuffer +// is constructed: the term buffer is fixed by _has_positions, and the gram-family decision that +// may reset _has_positions to false can only come from the provider. The second half of +// init() reuses the very provider created here. +Status SniiIndexColumnWriter::_create_analyzer_provider( + inverted_index::AnalyzerProviderPtr* analyzer_provider) { + try { + _char_string_reader = inverted_index::InvertedIndexAnalyzer::create_reader( + _analyzer_config.char_filter_map); + if (_should_analyzer) { + *analyzer_provider = inverted_index::InvertedIndexAnalyzer::create_analyzer_provider( + &_analyzer_config); + } + } catch (const CLuceneError& e) { + return Status::Error( + "SNII create analyzer failed: {}", e.what()); + } catch (const Exception& e) { + return Status::Error( + "SNII create analyzer failed: {}", e.what()); + } + _apply_gram_family_scheme(*analyzer_provider); + // After the scheme is settled and before any row arrives: the solve needs the scheme's + // max_gram, and holding rows back only works if it starts with the first one. + _arm_density_calibration(); + return Status::OK(); +} + Status SniiIndexColumnWriter::init() { _should_analyzer = inverted_index::InvertedIndexAnalyzer::should_analyzer(_index_meta->properties()); @@ -52,6 +124,20 @@ Status SniiIndexColumnWriter::init() { INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES; _config = _has_positions ? ::doris::snii::format::IndexConfig::kDocsPositions : ::doris::snii::format::IndexConfig::kDocsOnly; + _analyzer_config.analyzer_name = get_analyzer_name_from_properties(_index_meta->properties()); + _analyzer_config.parser_type = get_inverted_index_parser_type_from_string( + get_parser_string_from_properties(_index_meta->properties())); + _analyzer_config.parser_mode = + get_parser_mode_string_from_properties(_index_meta->properties()); + _analyzer_config.char_filter_map = + get_parser_char_filter_map_from_properties(_index_meta->properties()); + _analyzer_config.lower_case = + get_parser_lowercase_from_properties(_index_meta->properties()); + _analyzer_config.stop_words = get_parser_stopwords_from_properties(_index_meta->properties()); + // Reader, provider and the gram-family decision all happen here -- all of them before the + // term buffer is constructed below. + inverted_index::AnalyzerProviderPtr analyzer_provider; + RETURN_IF_ERROR(_create_analyzer_provider(&analyzer_provider)); auto ignore_above_value = get_parser_ignore_above_value_from_properties(_index_meta->properties()); _ignore_above = cast_set(std::stoul(ignore_above_value)); @@ -90,23 +176,8 @@ Status SniiIndexColumnWriter::init() { auto* global_limiter = ::doris::snii::writer::GlobalMemoryLimiter::instance(); global_limiter->set_min_victim_arena_bytes(config::snii_forced_spill_min_arena_bytes); _term_buffer->attach_global_limiter(global_limiter); - _analyzer_config.analyzer_name = get_analyzer_name_from_properties(_index_meta->properties()); - _analyzer_config.parser_type = get_inverted_index_parser_type_from_string( - get_parser_string_from_properties(_index_meta->properties())); - _analyzer_config.parser_mode = - get_parser_mode_string_from_properties(_index_meta->properties()); - _analyzer_config.char_filter_map = - get_parser_char_filter_map_from_properties(_index_meta->properties()); - _analyzer_config.lower_case = - get_parser_lowercase_from_properties(_index_meta->properties()); - _analyzer_config.stop_words = get_parser_stopwords_from_properties(_index_meta->properties()); try { - _char_string_reader = inverted_index::InvertedIndexAnalyzer::create_reader( - _analyzer_config.char_filter_map); if (_should_analyzer) { - auto analyzer_provider = - inverted_index::InvertedIndexAnalyzer::create_analyzer_provider( - &_analyzer_config); _analyzer = analyzer_provider->get_analyzer(); } } catch (const CLuceneError& e) { @@ -177,9 +248,8 @@ Status SniiIndexColumnWriter::_add_value_tokens(const Slice& value, uint32_t doc _char_string_reader->init(logical_value.data(), cast_set(logical_value.size()), false); { - std::unique_ptr owned_token_stream( - _analyzer->tokenStream(L"", _char_string_reader)); - auto* token_stream = owned_token_stream.get(); + std::unique_ptr owned_token_stream; + auto* token_stream = _plain_lane_token_stream(&owned_token_stream); // EXACT InvertedIndexAnalyzer::get_analyse_result semantics, // including the subtle one: an empty token's position increment is // dropped WITH the token (not accumulated into the next). @@ -193,7 +263,7 @@ Status SniiIndexColumnWriter::_add_value_tokens(const Slice& value, uint32_t doc consume_token(term, position, _has_positions); } } - token_stream->close(); + _finish_plain_lane_row(token_stream, owned_token_stream != nullptr); } } catch (const CLuceneError& e) { return _latch_analysis_failure(Status::Error( @@ -213,6 +283,20 @@ Status SniiIndexColumnWriter::add_values(const std::string /*name*/, const void* } const auto* v = reinterpret_cast(values); for (size_t i = 0; i < count; ++i) { + if (_density_calibrating) { + // Held back, not dropped: this row is tokenized once the rate is known. Feeding + // the solver here costs one linear pass over the row and retains nothing of it. + _density_solver->observe(std::string_view(v->data, v->size)); + _density_sample.emplace_back(_rid, std::string(v->data, v->size)); + _density_sample_bytes += static_cast(v->size); + _report_density_sample_capacity(); + ++v; + ++_rid; + if (_density_sample_bytes >= config::gram_index_density_sample_bytes) { + RETURN_IF_ERROR(_finish_density_calibration()); + } + continue; + } uint32_t max_position = 0; uint32_t token_count = 0; RETURN_IF_ERROR(_add_value_tokens(*v, _rid, 0, &max_position, &token_count)); @@ -229,6 +313,10 @@ Status SniiIndexColumnWriter::add_values(const std::string /*name*/, const void* Status SniiIndexColumnWriter::add_array_values(size_t field_size, const void* value_ptr, const uint8_t* nested_null_map, const uint8_t* offsets_ptr, size_t count) { + // Arrays are not sampled: settle the rate on whatever has been seen rather than hold array + // rows back as well, which would complicate the docid bookkeeping for a shape the gram + // family is not built for. + RETURN_IF_ERROR(_finish_density_calibration()); if (!_failure_status.ok()) { return _failure_status; } @@ -278,6 +366,140 @@ void SniiIndexColumnWriter::_report_null_docids_capacity(bool release_all) { } } +// Hand back the token stream for one row. The two ways to get one differ in ownership AND in +// reset: +// +// - tokenStream() runs create_components() on every call -- a fresh tokenizer object graph per +// row -- wraps it in a heap TokenStreamWrapper the caller owns, and resets it internally +// (custom_analyzer.cpp:149-155). +// - reusableTokenStream() runs create_components() once and hands back a NON-OWNING pointer into +// the analyzer's cached components, and does NOT reset (custom_analyzer.cpp:157-163). +// +// `owned` therefore receives the stream only on the first lane; on the second the analyzer keeps +// it and this writer must not delete it. +// +// The gram family takes the reusable lane. Its tokenizer carries per-row scratch sized by the row +// (the fold buffer, the boundary flags, the gram list, the dedupe table), and rebuilding all of it +// for every row was pure waste: reuse keeps the buffers warm and turns their capacity into a +// steady, reportable quantity (see _report_gram_buffers_capacity). Holding on to the analyzer's +// cached components is safe here because this writer built its own analyzer provider in init() +// and IndexPolicyMgr does not cache providers (index_policy_mgr.cpp:266-274), so the +// CustomAnalyzer -- and everything hanging off it -- belongs to this writer alone. +// +// Everything else stays on the owning lane: reuse changes the lifetime of an object shared by +// every custom analyzer, and the gram family is the only configuration whose cost justifies +// auditing that. +lucene::analysis::TokenStream* SniiIndexColumnWriter::_plain_lane_token_stream( + std::unique_ptr* owned) { + if (!_gram_scheme.has_value()) { + owned->reset(_analyzer->tokenStream(L"", _char_string_reader)); + return owned->get(); + } + auto* token_stream = _analyzer->reusableTokenStream(L"", _char_string_reader); + if (_gram_tokenizer == nullptr) { + // R22 keeps char filters and token filters out of the gram family, so the components' + // sink IS the tokenizer -- there is no filter wrapped around it to cast through. + _gram_tokenizer = dynamic_cast(token_stream); + DORIS_CHECK(_gram_tokenizer != nullptr); + if (_pending_density_permille.has_value()) { + // Before reset() below, which is what extracts: the first row through here is + // already a held-back one being replayed, and it must be cut at the solved rate. + _gram_tokenizer->set_density_permille(*_pending_density_permille); + _pending_density_permille.reset(); + } + } else { + DCHECK_EQ(static_cast(_gram_tokenizer), token_stream); + } + // MANDATORY on this lane. reusableTokenStream only parks the new reader in _in_pending; + // DorisTokenizer::reset() is what promotes it to _in, and GramTokenizer::reset() is what + // re-extracts and rewinds the cursor. Skip it and the row is tokenized as a repeat of the + // previous one. + token_stream->reset(); + return token_stream; +} + +// End of one row on the plain lane, the mirror of _plain_lane_token_stream. An owned stream is +// closed, as before. The reusable one is not: close() is teardown, and the next row will reset +// and reuse this stream. Instead its buffers, whose capacity has just settled for this row, +// are mirrored into the memory reporter. +void SniiIndexColumnWriter::_finish_plain_lane_row(lucene::analysis::TokenStream* token_stream, + bool owned) { + if (owned) { + token_stream->close(); + } else { + _report_gram_buffers_capacity(); + } +} + +// The gram tokenizer's scratch (fold buffer, boundary flags, gram list, dedupe table) is sized by +// the longest row this writer has seen and is never shrunk. They are ordinary heap allocations, +// so the process-wide MemTrackerLimiter already sees them through the allocation hook -- but this +// writer's own MemoryReporter did not, which meant the number driving its flush/spill decision +// understated real RSS. On a DENSE scheme the scratch tracks the row length closely, and rows on +// the analyzer lane are not bounded by ignore_above, so a single long row could move it by tens of +// megabytes. Only meaningful on the reusable lane: without it the tokenizer dies at the end of +// every row and there is no resident capacity to report. +// Solves this segment's density from the rows held back for it, applies it, and tokenizes +// them. Idempotent, and a no-op unless a gram scheme is in force with the feature enabled. +// +// It runs from three places, which is the point: whichever of "the sample is full", "an array +// arrived" or "the segment is being flushed" comes first, the rate is settled before a single +// row is cut. Every row of a segment must be cut at one rate, because the query side +// reconstructs a segment's grams from the single rate its metadata records. +Status SniiIndexColumnWriter::_finish_density_calibration() { + if (!_density_calibrating) { + return Status::OK(); + } + _density_calibrating = false; + const uint16_t solved = _density_solver->solve( + cast_set(config::gram_index_density_coverage_permille)); + if (_density_solver->observed_windows() > 0) { + _gram_scheme->density_permille = solved; + _pending_density_permille = solved; + } + // Free the histogram before replaying: it is 256 KB that nothing needs again. + _density_solver.reset(); + + for (auto& [docid, value] : _density_sample) { + uint32_t max_position = 0; + uint32_t token_count = 0; + const Slice slice(value.data(), value.size()); + RETURN_IF_ERROR(_add_value_tokens(slice, docid, 0, &max_position, &token_count)); + if (_writes_norms) { + _encoded_norms.push_back(::doris::snii::query::encode_norm(token_count)); + _report_encoded_norms_capacity(); + } + } + std::vector>().swap(_density_sample); + _density_sample_bytes = 0; + _report_density_sample_capacity(/*release_all=*/true); + return Status::OK(); +} + +// The held-back rows are real resident bytes between the first row and the solve, so the +// reporter that drives this writer's flush and spill decisions has to see them. +void SniiIndexColumnWriter::_report_density_sample_capacity(bool release_all) { + if (_memory_reporter == nullptr) { + return; + } + const int64_t now = release_all ? 0 : _density_sample_bytes; + if (now != _density_sample_charged_bytes) { + _memory_reporter->report(now - _density_sample_charged_bytes); + _density_sample_charged_bytes = now; + } +} + +void SniiIndexColumnWriter::_report_gram_buffers_capacity(bool release_all) { + if (_memory_reporter == nullptr || _gram_tokenizer == nullptr) { + return; + } + const int64_t now = release_all ? 0 : static_cast(_gram_tokenizer->reserved_bytes()); + if (now != _gram_buffers_charged_bytes) { + _memory_reporter->report(now - _gram_buffers_charged_bytes); + _gram_buffers_charged_bytes = now; + } +} + void SniiIndexColumnWriter::_report_encoded_norms_capacity(bool release_all) { if (_memory_reporter == nullptr) { return; @@ -350,8 +572,13 @@ Status SniiIndexColumnWriter::finish() { // flush-scoped); release the accumulation-phase charge so the retained // reporter (and the observation tracker behind it) balances to zero. _report_null_docids_capacity(/*release_all=*/true); + // A segment smaller than the sample budget reaches here still holding its rows back. Solve + // on what there is and tokenize them, so a short segment is indexed like any other -- and + // so `options.gram_scheme` below carries the rate its rows were actually cut at. + RETURN_IF_ERROR(_finish_density_calibration()); IndexFileWriter::SniiAddIndexOptions options {}; options.is_direct_load = _is_direct_load; + options.gram_scheme = _gram_scheme; if (_writes_norms) { DORIS_CHECK_EQ(_encoded_norms.size(), _rid); options.encoded_norms = std::move(_encoded_norms); @@ -360,6 +587,10 @@ Status SniiIndexColumnWriter::finish() { _index_meta, cast_set(_rid), std::move(_null_docids), _term_buffer.get(), _config, std::move(options), _memory_reporter.get()); _report_encoded_norms_capacity(/*release_all=*/true); + // The tokenizer's buffers outlive this call (they die with _analyzer), but the reporter is + // about to be handed to the index file writer and become a long-lived process-wide counter, + // so its accumulation-phase charges must balance to zero first. + _report_gram_buffers_capacity(/*release_all=*/true); RETURN_IF_ERROR(status); _index_file_writer->retain_snii_memory_reporter(std::move(_memory_reporter)); _term_buffer.reset(); @@ -379,6 +610,7 @@ void SniiIndexColumnWriter::close_on_error() { // Balance the observation-tracker mirror before dropping the reporter. _report_null_docids_capacity(/*release_all=*/true); _report_encoded_norms_capacity(/*release_all=*/true); + _report_gram_buffers_capacity(/*release_all=*/true); _memory_reporter.reset(); _null_docids.clear(); std::vector().swap(_encoded_norms); diff --git a/be/src/storage/index/snii/snii_index_writer.h b/be/src/storage/index/snii/snii_index_writer.h index ed54a4657bb4d2..e895d57f4d79b6 100644 --- a/be/src/storage/index/snii/snii_index_writer.h +++ b/be/src/storage/index/snii/snii_index_writer.h @@ -24,6 +24,12 @@ #include #include "storage/index/index_writer.h" +// AnalyzerProviderPtr is already in the include closure of inverted_index_parser.h (a path +// almost every TU pulls in), so spelling it out here only makes the dependency visible and adds +// no forward-closure cost. +#include "storage/index/inverted/analyzer/analyzer_provider.h" +#include "storage/index/inverted/gram/gram_density.h" +#include "storage/index/inverted/gram/gram_scheme.h" #include "storage/index/inverted/inverted_index_parser.h" #include "storage/index/inverted/query/query_info.h" #include "storage/index/inverted/util/reader.h" @@ -34,7 +40,12 @@ namespace lucene::analysis { class Analyzer; -} +class TokenStream; +} // namespace lucene::analysis + +namespace doris::segment_v2::inverted_index { +class GramTokenizer; +} // namespace doris::segment_v2::inverted_index namespace doris::segment_v2 { @@ -71,6 +82,7 @@ class SniiIndexColumnWriter final : public IndexColumnWriter { const std::vector& encoded_norms_for_test() const { return _encoded_norms; } ::doris::snii::format::IndexConfig config_for_test() const { return _config; } bool writes_norms_for_test() const { return _writes_norms; } + const std::optional& gram_scheme_for_test() const { return _gram_scheme; } void set_analysis_for_test(inverted_index::ReaderPtr reader, std::shared_ptr analyzer) { _should_analyzer = true; @@ -80,12 +92,37 @@ class SniiIndexColumnWriter final : public IndexColumnWriter { #endif private: + // The first half of init(): build the char filter reader and (when _should_analyzer) the one + // analyzer provider, then let _apply_gram_family_scheme settle the gram-family decision. Must + // be called before SpimiTermBuffer is constructed. A CLuceneError / Exception thrown here is + // uniformly turned into INVERTED_INDEX_ANALYZER_ERROR. + Status _create_analyzer_provider(inverted_index::AnalyzerProviderPtr* analyzer_provider); + // Take the gram scheme from the provider and apply the consequences of the gram family + // (forcing docs-only). + void _apply_gram_family_scheme(const inverted_index::AnalyzerProviderPtr& analyzer_provider); Status _add_value_tokens(const Slice& value, uint32_t docid, uint32_t position_base, uint32_t* max_position, uint32_t* token_count); + // One row's token stream, reset and ready to drain. `owned` receives it only when this + // writer must delete it; the gram lane's stream stays owned by _analyzer. See the + // definition for the ownership/reset contract. + lucene::analysis::TokenStream* _plain_lane_token_stream( + std::unique_ptr* owned); + // Mirror of the above, run once the row's tokens are consumed: closes an owned stream, or + // charges the reused gram tokenizer's settled buffer capacity to the memory reporter. + void _finish_plain_lane_row(lucene::analysis::TokenStream* token_stream, bool owned); // Mirrors _null_docids' capacity into _memory_reporter (delta-charged); // release_all zeroes the charge (finish() handoff / close_on_error()). void _report_null_docids_capacity(bool release_all = false); void _report_encoded_norms_capacity(bool release_all = false); + // Mirrors the reused gram tokenizer's buffer capacity into _memory_reporter + // (delta-charged), same shape as the two above. A no-op until the reusable + // lane has cached _gram_tokenizer. + void _report_gram_buffers_capacity(bool release_all = false); + // Solves the density from what has been held back, applies it, and tokenizes the held-back + // rows. Idempotent, and a no-op when the scheme is not gram family or the feature is off. + void _arm_density_calibration(); + Status _finish_density_calibration(); + void _report_density_sample_capacity(bool release_all = false); Status _latch_analysis_failure(Status status); IndexFileWriter* _index_file_writer = nullptr; @@ -108,9 +145,38 @@ class SniiIndexColumnWriter final : public IndexColumnWriter { uint32_t _ignore_above = 0; uint32_t _rid = 0; ::doris::snii::format::IndexConfig _config = ::doris::snii::format::IndexConfig::kDocsOnly; + // Scheme parameters of a gram-family analyzer (an ngram tokenizer with a mode property); + // obtained by _apply_gram_family_scheme, before the term buffer is constructed, from the + // analyzer provider this writer created itself. Once it holds a value, docs-only is forced. + // A built-in analyzer, an analyzer carrying filters, and an index carrying an index-level + // char_filter all leave it nullopt (R21/R22). + std::optional _gram_scheme; + // Adaptive density. The rate a gram scheme cuts at is solved from this segment's own bytes + // rather than taken from the tokenizer's configuration, because one configured rate cannot + // mean the same thing on two columns -- measured, a nominal 0.25 produced 0.204 to 0.287 + // grams per byte and 91.7% to 98.9% coverage across three corpora. + // + // The first rows of a segment are held back rather than tokenized: every row of a segment + // must be cut at ONE rate, since the query side reconstructs a segment's grams from the + // single rate recorded in its metadata. Once the sample budget is reached the rate is + // solved, applied, and the held-back rows are tokenized with it. + std::unique_ptr _density_solver; + std::vector> _density_sample; + int64_t _density_sample_bytes = 0; + int64_t _density_sample_charged_bytes = 0; + bool _density_calibrating = false; + // Set when the rate has been solved but the tokenizer has not been obtained yet; applied + // where the tokenizer is first cast, before it cuts anything. + std::optional _pending_density_permille; InvertedIndexAnalyzerConfig _analyzer_config; inverted_index::ReaderPtr _char_string_reader; std::shared_ptr _analyzer; + // Non-owning. The gram lane pulls its token stream from + // Analyzer::reusableTokenStream, which hands back a pointer into the cached + // components _analyzer owns, so this stays valid exactly as long as + // _analyzer does and must be declared after it. Null until the first + // analyzed row, and always null off the gram lane. + inverted_index::GramTokenizer* _gram_tokenizer = nullptr; std::unique_ptr<::doris::snii::writer::MemoryReporter> _memory_reporter; std::unique_ptr<::doris::snii::writer::SpimiTermBuffer> _term_buffer; std::vector _null_docids; @@ -122,6 +188,7 @@ class SniiIndexColumnWriter final : public IndexColumnWriter { // G09 limiter cannot see. int64_t _null_docids_charged_bytes = 0; int64_t _encoded_norms_charged_bytes = 0; + int64_t _gram_buffers_charged_bytes = 0; Status _failure_status = Status::OK(); }; diff --git a/be/src/storage/index/snii/writer/logical_index_writer.cpp b/be/src/storage/index/snii/writer/logical_index_writer.cpp index 9aec2a54a95c17..a9683c132dc546 100644 --- a/be/src/storage/index/snii/writer/logical_index_writer.cpp +++ b/be/src/storage/index/snii/writer/logical_index_writer.cpp @@ -106,6 +106,7 @@ class StreamingTermEncoder { uint64_t prx_base) : writer_(writer), postings_(postings), + stop_gram_df_threshold_(writer->stop_gram_df_threshold_), term_has_prx_(term_has_prx), buffer_(buffer), frq_base_(frq_base), @@ -141,7 +142,21 @@ class StreamingTermEncoder { testing::note_term_freq_scan(); #endif bool exhausted = false; - RETURN_IF_ERROR(fill(format::kAdaptiveWindowDfThreshold, &exhausted)); + // With stop-gram on, the first fill has to be large enough to settle the question + // outright. The decision is only free before a window is emitted: once posting + // bytes are appended to the region they cannot be taken back, so pulling + // threshold+1 documents up front buys an exact answer at zero wasted bytes. The + // buffer holds one term at a time, so the extra documents are a bounded cost. + const uint32_t first_fill = + stop_gram_df_threshold_ > 0 + ? std::max(format::kAdaptiveWindowDfThreshold, stop_gram_df_threshold_ + 1) + : format::kAdaptiveWindowDfThreshold; + RETURN_IF_ERROR(fill(first_fill, &exhausted)); + // total_docs_ is now min(first_fill, df), and first_fill > threshold, so this is + // true exactly when df > threshold. + if (stop_gram_df_threshold_ > 0 && total_docs_ > stop_gram_df_threshold_) { + return encode_dropped(entry, stats, exhausted); + } if (exhausted && buffer_->document_count() < format::kAdaptiveWindowDfThreshold) { entry->term = std::move(postings_->term); entry->df = total_docs_; @@ -168,6 +183,23 @@ class StreamingTermEncoder { } private: + // stop-gram. Drains the source without encoding anything: the posting bytes are the + // saving and are deliberately not written, but the source still has to be consumed + // because a streaming source shares one cursor with the terms that follow, and the df + // has to come out exact because it is what the reader's cost gate sees. The entry + // that remains carries the term key and the df, which is what keeps "matches every + // document" distinguishable from "absent from the dictionary". + Status encode_dropped(DictEntry* entry, FreqStats* stats, bool exhausted) { + while (!exhausted) { + RETURN_IF_ERROR(fill(format::kAdaptiveWindowDocs, &exhausted)); + } + entry->term = std::move(postings_->term); + entry->df = total_docs_; + entry->posting_dropped = true; + *stats = stats_; + return Status::OK(); + } + Status fill(uint32_t target_docs, bool* exhausted) { buffer_->clear_reuse(); position_offsets_.clear(); @@ -352,6 +384,7 @@ class StreamingTermEncoder { LogicalIndexWriter* writer_; StreamedTermPostings* postings_; + uint32_t stop_gram_df_threshold_ = 0; bool term_has_prx_ = false; TermPostingBuffer* buffer_ = nullptr; uint64_t frq_base_ = 0; @@ -401,6 +434,26 @@ FreqStats fuse_freq_stats_for_test(const std::vector& freqs) { } } // namespace testing +// The df above which a term enters the high-df digest, or 0 to build no digest at all. +// +// Two indexes get no digest, and for the same reason -- nothing would ever read it: +// +// - Anything that is not gram family. The digest exists solely so the gram query's cost +// gate can bound a node without reading df, and no other query path consults it. An +// ordinary full-text index would pay its bytes and its cycles for a structure it can +// never use. +// - A segment smaller than the divisor. Its floor would have to be clamped up to 1, at +// which point "common" means "occurs twice" and the digest fills with terms that bound +// nothing useful. The reader stands down at the same size (its ceiling lands at 0 and +// the gate falls back), and so does stop-gram. +uint32_t high_df_floor_for(const SniiIndexInput& in) { + if (!in.gram_scheme.has_value() || in.doc_count < format::kHighDfDigestDivisor) { + return 0; + } + return static_cast(static_cast(in.doc_count) / + format::kHighDfDigestDivisor); +} + LogicalIndexWriter::LogicalIndexWriter(const SniiIndexInput& in) : LogicalIndexWriter(in, TrackedNullDocids(std::vector(in.null_docids))) {} @@ -416,6 +469,9 @@ LogicalIndexWriter::LogicalIndexWriter(const SniiIndexInput& in, TrackedNullDoci terms_(in.terms), term_source_(in.term_source), encoded_norms_(in.encoded_norms), + gram_scheme_(in.gram_scheme), + stop_gram_df_threshold_(in.stop_gram_df_threshold), + high_df_floor_(high_df_floor_for(in)), target_dict_block_bytes_(in.target_dict_block_bytes != 0 ? in.target_dict_block_bytes : format::kDefaultTargetDictBlockBytes), @@ -505,6 +561,93 @@ struct LogicalIndexWriter::BlockState { // Out-of-line so unique_ptr sees the complete type (see header). LogicalIndexWriter::~LogicalIndexWriter() = default; +// Offers one term to the high-df digest. The digest keeps the highest-df terms seen, as a +// min-heap on df so the cheapest entry to evict is always at the front, which makes the +// whole pass O(terms log K) time and O(K) space -- no second sort over the vocabulary. +// +// Terms below the floor are dropped outright: the digest exists to let a query GIVE UP +// early, and a term rare enough to sit below the floor is one no cost gate would give up on. +void LogicalIndexWriter::note_high_df_term(uint64_t term_hash, uint32_t df) { + // A zero floor means no digest for this index (see high_df_floor_for), and it has to be + // tested separately: `df < 0` is never true, so the comparison below would let every + // term through on exactly the indexes that want none. + if (high_df_floor_ == 0 || df < high_df_floor_) { + return; + } + const auto by_df = [](const HighDfEntry& lhs, const HighDfEntry& rhs) { + return lhs.df > rhs.df; // greater-than gives a MIN-heap on df + }; + if (high_df_terms_.size() < format::kMaxHighDfDigestTerms) { + high_df_terms_.push_back({df, term_hash}); + std::ranges::push_heap(high_df_terms_, by_df); + return; + } + if (df <= high_df_terms_.front().df) { + return; + } + std::ranges::pop_heap(high_df_terms_, by_df); + high_df_terms_.back() = {df, term_hash}; + std::ranges::push_heap(high_df_terms_, by_df); +} + +// Turns the heap into the on-disk digest: ascending by hash so the reader can binary +// search, plus the ceiling that gives a miss its meaning. +void LogicalIndexWriter::finish_high_df_digest(format::HighDfTerms* out) const { + out->term_hash.clear(); + out->df.clear(); + if (high_df_terms_.empty()) { + // Nothing reached the floor, so every term in this index is below it. That is still + // a usable bound, and a strong one. + out->df_ceiling = high_df_floor_ == 0 ? 0 : high_df_floor_ - 1; + return; + } + + // The digest has to stay a rounding error against the index it describes, and the + // absolute cap alone does not deliver that on a small segment: there the floor is a + // handful of documents, most of the vocabulary clears it, and the cap fills with terms + // that are "common" only in the sense of appearing twice. Sized against the vocabulary + // as well, the digest stays proportional -- on a large index the df floor is the binding + // limit and this cap never engages, while on a small one it is what keeps a 45 KB digest + // off a 96 KB index. + const size_t cap = std::min(format::kMaxHighDfDigestTerms, + std::max(format::kMinHighDfDigestTerms, + static_cast(term_count_) / + format::kHighDfDigestVocabularyShare)); + std::vector sorted(high_df_terms_); + if (sorted.size() > cap) { + // Keep the `cap` largest by df; the largest one dropped becomes the bound for + // everything the digest no longer holds. + std::ranges::nth_element( + sorted, sorted.begin() + static_cast(cap), + [](const HighDfEntry& lhs, const HighDfEntry& rhs) { return lhs.df > rhs.df; }); + out->df_ceiling = sorted[cap].df; + sorted.resize(cap); + } else if (high_df_terms_.size() == format::kMaxHighDfDigestTerms) { + // The streaming cap truncated it, so the weakest entry kept is the best bound + // available for everything the heap evicted. + out->df_ceiling = high_df_terms_.front().df; + } else { + // Nothing above the floor was omitted at all. + out->df_ceiling = high_df_floor_ == 0 ? 0 : high_df_floor_ - 1; + } + + std::ranges::sort(sorted, [](const HighDfEntry& lhs, const HighDfEntry& rhs) { + return lhs.term_hash < rhs.term_hash; + }); + out->term_hash.reserve(sorted.size()); + out->df.reserve(sorted.size()); + for (const HighDfEntry& entry : sorted) { + // bsbf_hash collisions would put two terms at one key; keep the larger df so the + // digest stays an upper bound for both. + if (!out->term_hash.empty() && out->term_hash.back() == entry.term_hash) { + out->df.back() = std::max(out->df.back(), entry.df); + continue; + } + out->term_hash.push_back(entry.term_hash); + out->df.push_back(entry.df); + } +} + Status LogicalIndexWriter::process_term(StreamedTermPostings& tp, BlockState* st) { const bool term_has_prx = has_prx_ && tp.retain_positions; @@ -528,6 +671,7 @@ Status LogicalIndexWriter::process_term(StreamedTermPostings& tp, BlockState* st term_hashes_.push_back(term_hash); ++term_count_; stats_.sum_total_term_freq += stats.total_freq; + note_high_df_term(term_hash, entry.df); st->block->add_entry(std::move(entry)); if (st->block->estimated_bytes() >= target_dict_block_bytes_) { RETURN_IF_ERROR(flush_block(st->block.get(), st->block_first_term)); @@ -836,6 +980,8 @@ Status LogicalIndexWriter::finish_metadata(const SectionRefs& abs_refs, uint64_t core.index_config = index_config_; core.stats = stats_; core.section_refs = abs_refs; + core.gram_scheme = gram_scheme_; + finish_high_df_digest(&core.high_df_terms); ByteSink core_sink; RETURN_IF_ERROR(format::encode_core_metadata(core, &core_sink)); out->core = core_sink.take(); diff --git a/be/src/storage/index/snii/writer/logical_index_writer.h b/be/src/storage/index/snii/writer/logical_index_writer.h index 693d020cc6ede0..7e6e7875d76d6f 100644 --- a/be/src/storage/index/snii/writer/logical_index_writer.h +++ b/be/src/storage/index/snii/writer/logical_index_writer.h @@ -146,6 +146,21 @@ struct SniiIndexInput { // reporting. NEVER report live_bytes_ (a gated estimate); report // arena_bytes()+slot_of_+dict ram_bytes_. MemoryReporter* mem_reporter = nullptr; + // Present when the dictionary was generated by a gram-family tokenizer. + std::optional gram_scheme; + // stop-gram: a term whose df exceeds this many documents keeps its dictionary entry + // (term key + df, marked posting_dropped) but has no posting list written. 0 disables. + // + // The caller owns the policy, because the safety argument is the caller's: dropping a + // posting turns the term into one that matches every document, which is sound only + // where the index is an approximate filter whose candidates get re-checked. The + // caller is also the one that knows doc_count and can decide a row floor, so it hands + // down an absolute document count rather than a ratio. + // + // Measured motivation: on real log text the terms above the query-side cost gate's + // threshold are 0.7% of the vocabulary and hold 88.6% of the posting entries, and the + // gate already refuses to read every one of them. + uint32_t stop_gram_df_threshold = 0; }; // Move-only ownership of a NULL-docid allocation and its precharged bytes. @@ -359,6 +374,23 @@ class LogicalIndexWriter { SpimiTermBuffer* term_source_; // streaming source (null => use terms_) uint64_t term_count_ = 0; // distinct terms actually consumed const std::vector& encoded_norms_; + std::optional gram_scheme_; + // See SniiIndexInput::stop_gram_df_threshold. 0 leaves every posting list in place. + uint32_t stop_gram_df_threshold_ = 0; + + // High-df digest, built as terms stream past. See format::HighDfTerms for what a reader + // does with it, and note_high_df_term for how it is maintained. + struct HighDfEntry { + uint32_t df = 0; + uint64_t term_hash = 0; + }; + // Derived from this segment's own doc_count, so the digest scales with the index rather + // than with a configured constant: a small segment records the terms that are common + // FOR IT, which is the only sense in which "common" means anything to a cost gate. + uint32_t high_df_floor_ = 0; + std::vector high_df_terms_; // min-heap on df, capped + void note_high_df_term(uint64_t term_hash, uint32_t df); + void finish_high_df_digest(format::HighDfTerms* out) const; uint32_t target_dict_block_bytes_; // G16-h: zstd levels (dict whole-block / prx auto mode), from SniiIndexInput. diff --git a/be/test/storage/index/inverted/ananlyzer/custom_analyzer_test.cpp b/be/test/storage/index/inverted/ananlyzer/custom_analyzer_test.cpp index af81ce76477e57..0ca896ae7084b0 100644 --- a/be/test/storage/index/inverted/ananlyzer/custom_analyzer_test.cpp +++ b/be/test/storage/index/inverted/ananlyzer/custom_analyzer_test.cpp @@ -21,7 +21,10 @@ #include #include +#include +#include #include +#include #include "CLucene/store/Directory.h" #include "CLucene/store/FSDirectory.h" @@ -32,6 +35,7 @@ #include "storage/index/inverted/query/phrase_prefix_query.h" #include "storage/index/inverted/query/phrase_query.h" #include "storage/index/inverted/setting.h" +#include "storage/index/inverted/tokenizer/ngram/gram_tokenizer.h" CL_NS_USE(util) CL_NS_USE(store) @@ -197,6 +201,64 @@ TEST_F(CustomAnalyzerTest, CustomNgramAnalyzer) { } } +// The SNII writer's gram lane rests on two structural facts about this analyzer that no compiler +// checks. R22 keeps char filters and token filters out of the gram family, so the components' +// sink IS the tokenizer and the writer's dynamic_cast to GramTokenizer must succeed (it is a +// DORIS_CHECK in production, i.e. an abort). And reusableTokenStream must keep handing back the +// one cached stream, so the writer may cache that pointer for the whole segment. The third +// property below is the behavioural half: reusableTokenStream does not reset, so with the +// caller's explicit reset every row must come out exactly as a fresh owning stream would produce +// it, whatever the previous row was. +TEST_F(CustomAnalyzerTest, GramAnalyzerReusesOneTokenizerAcrossRows) { + Settings gram_params; + gram_params.set("mode", "sparse"); + gram_params.set("min_gram", "3"); + gram_params.set("max_gram", "16"); + gram_params.set("density", "0.25"); + + CustomAnalyzerConfig::Builder builder; + builder.with_tokenizer_config("ngram", gram_params); + auto analyzer = CustomAnalyzer::build_custom_analyzer(builder.build()); + + const std::vector rows {"rpc error: code = Unavailable desc = transport closing", + "手机微博", "SELECT COUNT(*) FROM tbl WHERE k = 42"}; + const auto drain = [](lucene::analysis::TokenStream* stream) { + std::vector tokens; + Token token; + while (stream->next(&token)) { + tokens.emplace_back(token.termBuffer(), token.termLength()); + } + return tokens; + }; + + auto reader = std::make_shared>(); + lucene::analysis::TokenStream* cached = nullptr; + std::vector> reused; + for (const auto& row : rows) { + reader->init(row.data(), static_cast(row.size()), false); + auto* stream = analyzer->reusableTokenStream(L"", reader); + if (cached == nullptr) { + cached = stream; + EXPECT_NE(dynamic_cast(stream), nullptr) + << "a gram analyzer must expose its tokenizer directly, with no filter on top"; + } else { + EXPECT_EQ(stream, cached) << "reusableTokenStream must reuse one cached stream"; + } + stream->reset(); // reusableTokenStream does not do this; the caller must + reused.push_back(drain(stream)); + } + + for (size_t i = 0; i < rows.size(); i++) { + reader->init(rows[i].data(), static_cast(rows[i].size()), false); + std::unique_ptr fresh(analyzer->tokenStream(L"", reader)); + const std::vector tokens = drain(fresh.get()); + fresh->close(); + // Row 1 is non-ASCII and yields nothing now that the index declines that text; + // reuse still has to agree with a fresh stream on it, which is the point here. + EXPECT_EQ(reused[i], tokens) << "row " << i; + } +} + TEST_F(CustomAnalyzerTest, TokenStreamNotSupported) { CustomAnalyzerConfig::Builder builder; builder.with_tokenizer_config("standard", {}); diff --git a/be/test/storage/index/inverted/tokenizer/gram_tokenizer_test.cpp b/be/test/storage/index/inverted/tokenizer/gram_tokenizer_test.cpp new file mode 100644 index 00000000000000..32e6b0853b85cb --- /dev/null +++ b/be/test/storage/index/inverted/tokenizer/gram_tokenizer_test.cpp @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/tokenizer/ngram/gram_tokenizer.h" + +#include + +#include +#include +#include + +#include "storage/index/inverted/setting.h" +#include "storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h" + +namespace doris::segment_v2 { + +using namespace inverted_index; + +// Ruling R8: the BE UT binary uses unity builds, and ngram_tokenizer_test.cpp already defines a +// tokenize(NGramTokenizerFactory&, const std::string&) with the same name and signature in this +// namespace, so reusing the name would be a redefinition error in the merged translation unit. +// The helper here lives in a file-private named namespace and is renamed to gram_tokenize, which +// avoids the clash twice over. +namespace gram_tokenizer_test_detail { + +std::vector gram_tokenize(NGramTokenizerFactory& factory, const std::string& data) { + auto tokenizer = factory.create(); + auto reader = std::make_shared>(); + reader->init(data.data(), data.size(), false); + tokenizer->set_reader(reader); + tokenizer->reset(); + std::vector out; + Token t; + while (tokenizer->next(&t)) { + out.emplace_back(t.termBuffer(), t.termLength()); + } + return out; +} + +} // namespace gram_tokenizer_test_detail + +using gram_tokenizer_test_detail::gram_tokenize; + +TEST(GramTokenizerTest, ModeAbsentKeepsLegacyBehaviour) { + NGramTokenizerFactory factory; + std::unordered_map args {{"min_gram", "2"}, {"max_gram", "3"}}; + Settings settings(args); + factory.initialize(settings); + EXPECT_FALSE(factory.gram_scheme().has_value()); + EXPECT_EQ(gram_tokenize(factory, "abc"), (std::vector {"ab", "abc", "bc"})); +} + +TEST(GramTokenizerTest, SparseModeProducesCdcGrams) { + NGramTokenizerFactory factory; + std::unordered_map args { + {"mode", "sparse"}, {"min_gram", "3"}, {"max_gram", "16"}, {"density", "0.25"}}; + Settings settings(args); + factory.initialize(settings); + ASSERT_TRUE(factory.gram_scheme().has_value()); + EXPECT_EQ(factory.gram_scheme()->mode, gram::GramMode::SPARSE); + // Same golden as GramExtractorTest.SparseGoldenFromPrototype (in start-position order) + EXPECT_EQ(gram_tokenize(factory, "rpc error: code = Unavailable"), + (std::vector {"or: co", "cod", "ode = U", " Unavai", "ailable"})); + // Non-ASCII text is not indexed: it yields no terms and separates the ASCII around it, + // and "ab" is shorter than min_gram, so this row contributes nothing at all. + EXPECT_EQ(gram_tokenize(factory, "手机ab微博"), (std::vector {})); + // The CJK is adjacent to the ASCII on purpose: the ASCII segment is then byte-for-byte + // the string above, so this pins that skipping the non-ASCII leaves the segment's own + // gram boundaries untouched rather than shifting them. + EXPECT_EQ(gram_tokenize(factory, "手机rpc error: code = Unavailable"), + (std::vector {"or: co", "cod", "ode = U", " Unavai", "ailable"})) + << "the ASCII part of a mixed row indexes exactly as it would alone"; + EXPECT_EQ(gram_tokenize(factory, ""), (std::vector {})); +} + +TEST(GramTokenizerTest, DenseModeAndAutoAlias) { + NGramTokenizerFactory factory; + std::unordered_map args {{"mode", "dense"}, {"min_gram", "3"}}; + factory.initialize(Settings(args)); + EXPECT_EQ(gram_tokenize(factory, "abcd"), (std::vector {"abc", "bcd"})); + NGramTokenizerFactory f2; + std::unordered_map a2 {{"mode", "auto"}}; + f2.initialize(Settings(a2)); + EXPECT_EQ(f2.gram_scheme()->mode, gram::GramMode::SPARSE); // P0: auto = sparse +} + +// When min/max_gram are absent the defaults come from GramScheme's member initializers (3/16); +// the factory no longer injects a copy of its own. +TEST(GramTokenizerTest, GramDefaultsComeFromGramScheme) { + NGramTokenizerFactory factory; + std::unordered_map args {{"mode", "sparse"}}; + factory.initialize(Settings(args)); + ASSERT_TRUE(factory.gram_scheme().has_value()); + EXPECT_EQ(factory.gram_scheme()->min_len, 3U); + EXPECT_EQ(factory.gram_scheme()->max_len, 16U); +} + +// With lower_case=true the folding happens before the split (the boundary hash is computed over +// the folded bytes as well), so the same text in different cases must yield exactly the same gram +// sequence. +TEST(GramTokenizerTest, LowerCaseFoldsBeforeExtraction) { + NGramTokenizerFactory factory; + std::unordered_map args {{"mode", "sparse"}, {"lower_case", "true"}}; + factory.initialize(Settings(args)); + ASSERT_TRUE(factory.gram_scheme().has_value()); + EXPECT_TRUE(factory.gram_scheme()->lower_case); + EXPECT_EQ(gram_tokenize(factory, "Code = Unavailable"), + gram_tokenize(factory, "code = unavailable")); + // Use a longer input that is guaranteed to yield grams, so the check above cannot pass + // vacuously with both sides empty. + const auto mixed = gram_tokenize(factory, "RPC error: Code = Unavailable"); + EXPECT_FALSE(mixed.empty()); + EXPECT_EQ(mixed, gram_tokenize(factory, "rpc error: code = unavailable")); +} + +TEST(GramTokenizerTest, GramModeSkipsLegacyMinMaxGapCheck) { + // Today initialize throws INVALID_ARGUMENT for max_gram-min_gram>1 + // (ngram_tokenizer_factory.cpp:29-36); the gram family returns from inside the mode branch and + // is not subject to that limit. + NGramTokenizerFactory factory; + std::unordered_map args { + {"mode", "sparse"}, {"min_gram", "3"}, {"max_gram", "24"}}; + EXPECT_NO_THROW(factory.initialize(Settings(args))); + std::unordered_map bad {{"mode", "sparse"}, {"density", "2"}}; + EXPECT_THROW(NGramTokenizerFactory().initialize(Settings(bad)), Exception); +} + +} // namespace doris::segment_v2 diff --git a/be/test/storage/index/snii/format/core_metadata_test.cpp b/be/test/storage/index/snii/format/core_metadata_test.cpp index ce717fe9477d72..c0476e5aeee508 100644 --- a/be/test/storage/index/snii/format/core_metadata_test.cpp +++ b/be/test/storage/index/snii/format/core_metadata_test.cpp @@ -26,6 +26,7 @@ #include #include "gen_cpp/snii.pb.h" +#include "storage/index/inverted/gram/gram_scheme.h" #include "storage/index/snii/encoding/byte_sink.h" #include "storage/index/snii/encoding/byte_source.h" #include "storage/index/snii/encoding/section_framer.h" @@ -33,6 +34,9 @@ namespace doris::snii::format { namespace { +using segment_v2::gram::GramMode; +using segment_v2::gram::GramScheme; + CoreMetadata sample_core(IndexConfig index_config = IndexConfig::kDocsOnly) { CoreMetadata metadata; metadata.index_config = index_config; @@ -107,6 +111,7 @@ void expect_core_eq(const CoreMetadata& expected, const CoreMetadata& actual) { EXPECT_EQ(expected.section_refs.null_bitmap.length, actual.section_refs.null_bitmap.length); EXPECT_EQ(expected.section_refs.bsbf.offset, actual.section_refs.bsbf.offset); EXPECT_EQ(expected.section_refs.bsbf.length, actual.section_refs.bsbf.length); + EXPECT_EQ(expected.gram_scheme, actual.gram_scheme); } // Since A2, core metadata forbids norms on docs-only segments: norms require positions. @@ -300,6 +305,71 @@ TEST(SniiCoreMetadata, RejectsBadFrameTypeCrcAndTruncation) { .is()); } +TEST(SniiCoreMetadata, GramSchemeRoundTrip) { + auto expected = sample_core(); + GramScheme scheme; + scheme.mode = GramMode::DENSE; + scheme.min_len = 3; + scheme.max_len = 3; + scheme.density_permille = 1000; + scheme.lower_case = true; + scheme.hash_version = 1; + expected.gram_scheme = scheme; + + CoreMetadata actual; + ASSERT_TRUE(decode_core_metadata(Slice(encode(expected)), &actual).ok()); + ASSERT_TRUE(actual.gram_scheme.has_value()); + EXPECT_TRUE(*actual.gram_scheme == scheme); + expect_core_eq(expected, actual); + + // A non-gram index has no tokenizer scheme to persist. + const auto none_framed = encode(sample_core()); + CoreMetadata none_actual; + ASSERT_TRUE(decode_core_metadata(Slice(none_framed), &none_actual).ok()); + EXPECT_FALSE(none_actual.gram_scheme.has_value()); + + const auto none_payload = payload_of(none_framed); + doris::snii::SniiCoreMetadataPB none_pb; + ASSERT_TRUE(none_pb.ParseFromArray(none_payload.data(), static_cast(none_payload.size()))); + EXPECT_FALSE(none_pb.has_gram_scheme()); +} + +TEST(SniiCoreMetadata, GramSchemeSparseRoundTrip) { + auto expected = sample_core(); + expected.gram_scheme = GramScheme {}; // member defaults: SPARSE / 3 / 16 / 250 / lc0 / v1 + + CoreMetadata actual; + ASSERT_TRUE(decode_core_metadata(Slice(encode(expected)), &actual).ok()); + ASSERT_TRUE(actual.gram_scheme.has_value()); + EXPECT_EQ(actual.gram_scheme->mode, GramMode::SPARSE); + EXPECT_EQ(actual.gram_scheme->min_len, 3U); + EXPECT_EQ(actual.gram_scheme->max_len, 16U); + EXPECT_EQ(actual.gram_scheme->density_permille, 250U); + expect_core_eq(expected, actual); +} + +// Decoding has to validate the whole scheme, not just glance at mode: an out-of-range scheme that +// slips through would feed values such as min_len=0 straight into GramExtractor. +TEST(SniiCoreMetadata, RejectsCorruptGramScheme) { + auto metadata = sample_core(); + metadata.gram_scheme = GramScheme {}; + + // mode only allows 1 (DENSE) / 2 (SPARSE). + const auto bad_mode = mutate_core_payload( + metadata, [](auto* core) { core->mutable_gram_scheme()->set_mode(7); }); + CoreMetadata actual; + auto status = decode_core_metadata(Slice(frame_payload(bad_mode)), &actual); + EXPECT_TRUE(status.is()) << status; + + // A partial message: only mode is set, every other field defaults to 0 by PB semantics, and 0 + // is not part of any valid scheme. + const auto partial = mutate_core_payload(sample_core(), [](auto* core) { + core->mutable_gram_scheme()->set_mode(static_cast(GramMode::SPARSE)); + }); + status = decode_core_metadata(Slice(frame_payload(partial)), &actual); + EXPECT_TRUE(status.is()) << status; +} + TEST(SniiCoreMetadata, ResetsOutputBeforeDecodeFailureAndNullOutputIsInvalid) { auto populated = sample_core(IndexConfig::kDocsPositions); CoreMetadata reused; diff --git a/be/test/storage/index/snii/format/dict_entry_test.cpp b/be/test/storage/index/snii/format/dict_entry_test.cpp index e80c75c5a42a44..0c3fac2a0884bf 100644 --- a/be/test/storage/index/snii/format/dict_entry_test.cpp +++ b/be/test/storage/index/snii/format/dict_entry_test.cpp @@ -302,3 +302,73 @@ TEST(SniiDictEntry, RejectPrefixLongerThanPrev) { Status s = decode_dict_entry(&src, "ab", IndexTier::kT1, &out); EXPECT_FALSE(s.ok()); } + +// ---- stop-gram: an entry that keeps its term and df but drops its posting ---- +// +// A gram whose df is high enough that the query-side cost gate refuses to read its +// posting list still costs that list on disk and in S3. Dropping the list while keeping +// the dictionary entry is what makes the saving available, and keeping the entry is what +// lets a reader tell "this gram matches everything" from "this gram is absent" -- the +// second is a strong filter, and mistaking the first for it drops matching rows. + +TEST(SniiDictEntry, PostingDroppedRoundTripKeepsTermAndDf) { + DictEntry in; + in.term = "the"; + in.df = 987654; + in.posting_dropped = true; + DictEntry out = RoundTrip(in, "", IndexTier::kT1); + EXPECT_EQ(out.term, in.term); + EXPECT_EQ(out.df, in.df); + EXPECT_TRUE(out.posting_dropped); +} + +// The point of the flag is size, so the encoding must actually drop the locator: a +// dropped entry is strictly shorter than the same term carrying a posting. +TEST(SniiDictEntry, PostingDroppedEncodesShorterThanALocator) { + DictEntry with_posting = MakePodRefSlim("common", 60000); + DictEntry dropped; + dropped.term = with_posting.term; + dropped.df = with_posting.df; + dropped.posting_dropped = true; + + ByteSink a; + ASSERT_TRUE(encode_dict_entry(with_posting, "", IndexTier::kT1, &a).ok()); + ByteSink b; + ASSERT_TRUE(encode_dict_entry(dropped, "", IndexTier::kT1, &b).ok()); + EXPECT_LT(b.view().size(), a.view().size()); +} + +// Tier 2 entries also carry a prx locator; a dropped posting writes neither locator, so the +// entry is df-only there as well. +TEST(SniiDictEntry, PostingDroppedRoundTripAtTier2) { + DictEntry in; + in.term = "and"; + in.df = 500000; + in.posting_dropped = true; + DictEntry out = RoundTrip(in, "", IndexTier::kT2); + EXPECT_EQ(out.term, in.term); + EXPECT_EQ(out.df, in.df); + EXPECT_TRUE(out.posting_dropped); + EXPECT_EQ(out.frq_len, 0U); + EXPECT_EQ(out.prx_len, 0U); +} + +// An ordinary entry must not come back marked, or every term would read as matching all. +TEST(SniiDictEntry, OrdinaryEntryIsNotMarkedDropped) { + DictEntry out = RoundTrip(MakePodRefSlim("plain", 3), "", IndexTier::kT1); + EXPECT_FALSE(out.posting_dropped); + DictEntry inl = RoundTrip(MakeInline("inline", 3), "", IndexTier::kT2); + EXPECT_FALSE(inl.posting_dropped); +} + +// Front coding still applies: a dropped entry sits in a block like any other, so the +// term key must survive being encoded against a shared prefix. +TEST(SniiDictEntry, PostingDroppedParticipatesInFrontCoding) { + DictEntry dropped; + dropped.term = "prefix_common"; + dropped.df = 70000; + dropped.posting_dropped = true; + DictEntry out = RoundTrip(dropped, "prefix_c", IndexTier::kT1); + EXPECT_EQ(out.term, "prefix_common"); + EXPECT_TRUE(out.posting_dropped); +} diff --git a/be/test/storage/index/snii/format/gram_index_section_sizes_test.cpp b/be/test/storage/index/snii/format/gram_index_section_sizes_test.cpp new file mode 100644 index 00000000000000..d3452309e57947 --- /dev/null +++ b/be/test/storage/index/snii/format/gram_index_section_sizes_test.cpp @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Where do the bytes of a gram index actually go? +// +// Measured on a cluster, a gram index runs about 1.0x to 1.4x the size of the column data +// it accelerates, and lowering the boundary density or the maximum gram length moves that +// only so far. Deciding whether the remaining size is worth attacking, and where, needs the +// split between the two regions that hold it: the term dictionary (already front-coded, +// with a full-term anchor every anchor_interval entries) and the postings. +// +// This builds real SNII segments over one corpus under several schemes and reports the +// section sizes the writer produced, straight out of the core metadata the reader reads. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "io/fs/local_file_system.h" +#include "runtime/exec_env.h" +#include "runtime/index_policy/index_policy_mgr.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/snii_index_writer.h" +#include "storage/tablet/tablet_schema.h" + +namespace doris::segment_v2 { +namespace { + +constexpr const char* kDir = "./ut_dir/gram_index_section_sizes"; +constexpr int64_t kIndexId = 4801; + +void assert_ok(const Status& st) { + ASSERT_TRUE(st.ok()) << st.to_string(); +} + +// The region split has to be measured on the vocabulary a real workload produces. +// GRAM_SECTION_CORPUS names a file of one row per line and GRAM_SECTION_ROWS caps how many +// are read; without the variable the case skips rather than fall back to generated rows. +// A generated corpus is not a substitute here: a templated one yields an order of magnitude +// fewer distinct terms than real log text, and the dictionary is sized by exactly that. +std::vector LoadCorpusFile(const char* path, size_t max_rows) { + std::vector out; + std::ifstream in(path); + std::string line; + while (out.size() < max_rows && std::getline(in, line)) { + if (!line.empty()) { + out.push_back(line); + } + } + return out; +} + +TabletIndex MakeIndexMeta(const std::map& props) { + TabletIndexPB pb; + pb.set_index_id(kIndexId); + pb.set_index_name("idx_gram_sections"); + pb.set_index_type(IndexType::INVERTED); + pb.add_col_unique_id(0); + auto& m = *pb.mutable_properties(); + for (const auto& [k, v] : props) { + m[k] = v; + } + TabletIndex idx; + idx.init_from_pb(pb); + return idx; +} + +struct Sections { + uint64_t dict = 0; + uint64_t posting = 0; + uint64_t null_bitmap = 0; + uint64_t bsbf = 0; + uint64_t file = 0; + uint64_t terms = 0; +}; + +} // namespace + +// Prints the region split for a few schemes. The only assertions are structural, so the +// case reports numbers without becoming a tripwire for ordinary size drift. +TEST(GramIndexSectionSizesTest, ReportsDictAndPostingSplit) { + std::filesystem::remove_all(kDir); + std::filesystem::create_directories(kDir); + + struct Cfg { + std::string name; + std::string mode; + std::string min_gram; + std::string max_gram; + std::string density; + }; + // density and max_gram act on different regions, so the grid crosses them: density + // changes how many grams are emitted, which sizes both regions, while max_gram + // changes how many distinct ones there are, which sizes the dictionary alone. + std::vector cfgs; + for (const std::string& d : {"0.25", "0.10", "0.05"}) { + for (const std::string& m : {"16", "8", "4"}) { + cfgs.push_back({"sparse d=" + d + " max=" + m, "sparse", "3", m, d}); + } + } + cfgs.push_back({"dense min=3", "dense", "3", "16", "0.25"}); + + const char* corpus_path = std::getenv("GRAM_SECTION_CORPUS"); + if (corpus_path == nullptr) { + GTEST_SKIP() << "set GRAM_SECTION_CORPUS to a corpus file (one row per line) to run; " + "there is no built-in corpus: a synthetic one is far too templated " + "to carry a size conclusion"; + } + const char* rows_env = std::getenv("GRAM_SECTION_ROWS"); + const size_t max_rows = rows_env ? static_cast(std::atoll(rows_env)) : 200000; + const std::vector corpus_rows = LoadCorpusFile(corpus_path, max_rows); + ASSERT_FALSE(corpus_rows.empty()) << "no rows read from " << corpus_path; + std::vector> corpus; + corpus.reserve(corpus_rows.size()); + uint64_t raw_bytes = 0; + for (const std::string& r : corpus_rows) { + raw_bytes += r.size(); + corpus.emplace_back(r); + } + + printf("\ncorpus: %s, %zu rows, %.2f MB raw\n", corpus_path, corpus.size(), + static_cast(raw_bytes) / 1048576.0); + printf("%-24s %10s %10s %10s %7s %7s %9s %9s\n", "scheme", "dict", "posting", "index", "dict%", + "post%", "terms", "idx/raw"); + + int policy_id = 9400; + for (const Cfg& cfg : cfgs) { + // SniiIndexColumnWriter takes its gram scheme from the analyzer provider, so the + // tokenizer parameters have to arrive as a real index policy; properties dropped on + // the index itself are ignored and every scheme would come out identical. + IndexPolicyMgr manager; + auto* exec_env = ExecEnv::GetInstance(); + IndexPolicyMgr* previous = exec_env->index_policy_mgr(); + exec_env->_index_policy_mgr = &manager; + + TIndexPolicy tok; + tok.id = ++policy_id; + tok.name = std::string("sec_tok_") + std::to_string(policy_id); + tok.type = TIndexPolicyType::TOKENIZER; + tok.properties["type"] = "ngram"; + tok.properties["mode"] = cfg.mode; + tok.properties["min_gram"] = cfg.min_gram; + tok.properties["max_gram"] = cfg.max_gram; + if (cfg.mode == "sparse") { + tok.properties["density"] = cfg.density; + } + TIndexPolicy ana; + ana.id = ++policy_id; + ana.name = std::string("sec_ana_") + std::to_string(policy_id); + ana.type = TIndexPolicyType::ANALYZER; + ana.properties["tokenizer"] = tok.name; + manager.apply_policy_changes({tok, ana}, {}); + + const TabletIndex index_meta = + MakeIndexMeta({{"analyzer", ana.name}, {"support_phrase", "false"}}); + const std::string path = std::string(kDir) + "/s" + std::to_string(policy_id) + ".idx"; + + io::FileWriterPtr fw; + assert_ok(io::global_local_filesystem()->create_file(path, &fw)); + IndexFileWriter ifw(io::global_local_filesystem(), path, "gram_sections_rowset", + /*seg_id=*/0, InvertedIndexStorageFormatPB::SNII, std::move(fw), + /*can_use_ram_dir=*/true, /*tablet_id=*/9301); + SniiIndexColumnWriter writer(&ifw, &index_meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + if (!writer.init().ok()) { + printf("%-24s (scheme rejected by the writer, skipped)\n", cfg.name.c_str()); + exec_env->_index_policy_mgr = previous; + continue; + } + std::vector slices; + slices.reserve(corpus.size()); + for (const auto& r : corpus) { + slices.emplace_back(*r); + } + assert_ok(writer.add_values("c1", slices.data(), slices.size())); + assert_ok(writer.finish()); + assert_ok(ifw.begin_close()); + assert_ok(ifw.finish_close()); + + doris::snii::io::LocalFileReader file; + assert_ok(file.open(path)); + doris::snii::reader::SniiSegmentReader segment; + assert_ok(doris::snii::reader::SniiSegmentReader::open(&file, &segment)); + doris::snii::reader::LogicalIndexReader index; + assert_ok(segment.open_index(static_cast(kIndexId), "", &index)); + + Sections s; + const auto& refs = index.section_refs(); + s.dict = refs.dict_region.length; + s.posting = refs.posting_region.length; + s.null_bitmap = refs.null_bitmap.length; + s.bsbf = refs.bsbf.length; + s.terms = index.stats().term_count; + s.file = std::filesystem::file_size(path); + + const double total = static_cast(s.dict + s.posting); + printf("%-24s %10llu %10llu %10llu %6.1f%% %6.1f%% %9llu %9.3f\n", cfg.name.c_str(), + static_cast(s.dict), static_cast(s.posting), + static_cast(s.file), + total > 0 ? 100.0 * static_cast(s.dict) / total : 0.0, + total > 0 ? 100.0 * static_cast(s.posting) / total : 0.0, + static_cast(s.terms), + raw_bytes > 0 ? static_cast(s.file) / static_cast(raw_bytes) : 0.0); + + EXPECT_GT(s.dict, 0U) << cfg.name; + EXPECT_GT(s.posting, 0U) << cfg.name; + EXPECT_GT(s.terms, 0U) << cfg.name; + exec_env->_index_policy_mgr = previous; + } + + std::filesystem::remove_all(kDir); +} + +} // namespace doris::segment_v2 diff --git a/be/test/storage/index/snii/format/gram_stop_gram_curve_test.cpp b/be/test/storage/index/snii/format/gram_stop_gram_curve_test.cpp new file mode 100644 index 00000000000000..931d424d2a6db3 --- /dev/null +++ b/be/test/storage/index/snii/format/gram_stop_gram_curve_test.cpp @@ -0,0 +1,318 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// What does each df cut-off actually buy? +// +// The stop-gram threshold has twice been chosen as a ratio of the segment -- 0.15%, then a +// tenth -- and both times the number came from an argument rather than from the corpus. The +// two arguments reach thresholds 66x apart, which is a sign that neither is derived from +// anything the data says. +// +// This sweeps the cut-off across the whole range on a real corpus and reports what each one +// costs and buys: how many terms it drops, what share of the posting entries those terms +// held, and what the index actually weighs afterwards. A rule that adapts to the segment has +// to be read off this curve, and the shape differs by script -- English log text puts almost +// all of its posting mass in a few hundred terms, while a CJK vocabulary spreads it out -- +// so the same rule must produce different cut-offs on the two, which is the whole point. +// +// Corpus-driven and skipped without one: +// GRAM_CURVE_CORPUS=/path/to/lines.txt GRAM_CURVE_ROWS=200000 +// (optional) GRAM_CURVE_JSON_FIELD=body to pull one field out of JSON lines. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/inverted/gram/gram_extractor.h" +#include "storage/index/inverted/gram/gram_scheme.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii_query_test_util.h" + +namespace doris::snii { +namespace { + +using snii_test::assert_ok; +using snii_test::make_term; +using snii_test::MemoryFile; +using snii_test::PostingDoc; + +std::vector read_corpus(const char* path, size_t rows) { + std::vector out; + std::ifstream in(path); + std::string line; + while (out.size() < rows && std::getline(in, line)) { + if (!line.empty()) { + out.push_back(line); + } + } + return out; +} + +// term -> sorted docids, built by running the real extractor over the corpus. +struct Corpus { + std::vector terms; + uint32_t doc_count = 0; + uint64_t raw_bytes = 0; + uint64_t posting_entries = 0; +}; + +// How the non-ASCII runs are turned into terms. The extractor emits one term per code +// point today (kUni); the other two are what a word-shaped scheme would produce, and the +// point of measuring them is that a Chinese word is rare where its characters are not. +// kNeededBi: keep a character term when it is rare enough to survive the cut, and emit a +// pair term only where BOTH characters are above it -- those are the positions where a query +// would otherwise have no rare handle at all. Purely distributional: the cut-off decides what +// "common" means, so the scheme adapts to the corpus rather than to a script. +enum class CjkMode { kUni, kBi, kUniBi, kNeededBi }; + +size_t codepoint_len_at(const std::string& s, size_t i) { + const auto c = static_cast(s[i]); + size_t len = 1; + if ((c >> 5) == 0x6) { + len = 2; + } else if ((c >> 4) == 0xE) { + len = 3; + } else if ((c >> 3) == 0x1E) { + len = 4; + } + return i + len > s.size() ? 1 : len; +} + +// Terms of one row: ASCII segments through the real extractor, non-ASCII runs through the +// mode under test. Deduplicated per row, exactly as the writer sees them. +void row_terms(const std::string& line, segment_v2::gram::GramExtractor& extractor, CjkMode mode, + const std::set* common, std::vector* out) { + out->clear(); + std::vector grams; + size_t i = 0; + const size_t L = line.size(); + while (i < L) { + if (static_cast(line[i]) < 0x80) { + size_t j = i; + while (j < L && static_cast(line[j]) < 0x80) { + j++; + } + // The extractor hands back views INTO the text it was given, so the segment has + // to outlive them; a temporary from substr() would leave every view dangling. + const std::string segment = line.substr(i, j - i); + extractor.extract(segment, &grams); + for (const std::string_view g : grams) { + out->emplace_back(g); + } + i = j; + } else { + std::vector run; + while (i < L && static_cast(line[i]) >= 0x80) { + const size_t len = codepoint_len_at(line, i); + run.push_back(line.substr(i, len)); + i += len; + } + if (mode == CjkMode::kUni || mode == CjkMode::kUniBi) { + for (const std::string& cp : run) { + out->push_back(cp); + } + } + if (mode == CjkMode::kBi || mode == CjkMode::kUniBi) { + for (size_t k = 0; k + 1 < run.size(); ++k) { + out->push_back(run[k] + run[k + 1]); + } + if (run.size() == 1 && mode == CjkMode::kBi) { + out->push_back(run[0]); // a lone character has no pair to form + } + } + if (mode == CjkMode::kNeededBi) { + for (const std::string& cp : run) { + if (common == nullptr || !common->contains(cp)) { + out->push_back(cp); + } + } + for (size_t k = 0; k + 1 < run.size(); ++k) { + if (common != nullptr && common->contains(run[k]) && + common->contains(run[k + 1])) { + out->push_back(run[k] + run[k + 1]); + } + } + } + } + } + std::ranges::sort(*out); + out->erase(std::unique(out->begin(), out->end()), out->end()); +} + +Corpus build_corpus(const std::vector& lines, + const segment_v2::gram::GramScheme& scheme, CjkMode mode, + const std::set* common = nullptr) { + segment_v2::gram::GramExtractor extractor(scheme); + std::map> by_term; + Corpus corpus; + corpus.doc_count = static_cast(lines.size()); + std::vector terms; + for (uint32_t docid = 0; docid < lines.size(); ++docid) { + corpus.raw_bytes += lines[docid].size(); + row_terms(lines[docid], extractor, mode, common, &terms); + for (const std::string& term : terms) { + by_term[term].push_back(PostingDoc {.docid = docid, .positions = {}}); + } + } + corpus.terms.reserve(by_term.size()); + for (auto& [term, docs] : by_term) { + corpus.posting_entries += docs.size(); + corpus.terms.push_back(make_term(term, std::move(docs))); + } + return corpus; +} + +struct Built { + uint64_t index_bytes = 0; + uint64_t dict_bytes = 0; + uint64_t posting_bytes = 0; + uint64_t terms_dropped = 0; + uint64_t entries_dropped = 0; +}; + +Built build_at(const Corpus& corpus, const segment_v2::gram::GramScheme& scheme, + uint32_t threshold) { + writer::SniiIndexInput input; + input.index_id = 77; + input.index_suffix = "body"; + input.config = format::IndexConfig::kDocsOnly; + input.doc_count = corpus.doc_count; + input.terms = corpus.terms; + input.gram_scheme = scheme; + input.stop_gram_df_threshold = threshold; + + Built built; + for (const writer::TermPostings& t : corpus.terms) { + if (threshold > 0 && t.docids.size() > threshold) { + built.terms_dropped++; + built.entries_dropped += t.docids.size(); + } + } + + MemoryFile file; + writer::SniiCompoundWriter compound(&file); + assert_ok(compound.add_logical_index(input)); + assert_ok(compound.finish()); + + reader::SniiSegmentReader segment; + assert_ok(reader::SniiSegmentReader::open(&file, &segment)); + reader::LogicalIndexReader index; + assert_ok(segment.open_index(77, "body", &index)); + const auto& refs = index.section_refs(); + built.dict_bytes = refs.dict_region.length; + built.posting_bytes = refs.posting_region.length; + built.index_bytes = file.size(); + return built; +} + +} // namespace + +TEST(GramStopGramCurveTest, ReportsWhatEachCutOffBuys) { + const char* path = std::getenv("GRAM_CURVE_CORPUS"); + if (path == nullptr) { + GTEST_SKIP() << "set GRAM_CURVE_CORPUS to a line-per-record corpus"; + } + const char* rows_env = std::getenv("GRAM_CURVE_ROWS"); + const size_t rows = rows_env != nullptr ? std::strtoul(rows_env, nullptr, 10) : 200000; + const std::vector lines = read_corpus(path, rows); + ASSERT_FALSE(lines.empty()) << "corpus " << path << " is empty or unreadable"; + + segment_v2::gram::GramScheme scheme; + scheme.mode = segment_v2::gram::GramMode::SPARSE; + scheme.min_len = 3; + scheme.max_len = 4; + scheme.density_permille = 250; + + const char* mode_env = std::getenv("GRAM_CURVE_CJK"); + const std::string mode_name = mode_env != nullptr ? mode_env : "uni"; + CjkMode mode = CjkMode::kUni; + if (mode_name == "bi") { + mode = CjkMode::kBi; + } else if (mode_name == "uni+bi") { + mode = CjkMode::kUniBi; + } else if (mode_name == "needed-bi") { + mode = CjkMode::kNeededBi; + } + const Corpus corpus = build_corpus(lines, scheme, mode); + printf("cjk mode: %s\n", mode_name.c_str()); + if (mode == CjkMode::kNeededBi) { + // The scheme depends on which terms the cut-off calls common, so each cut-off gets + // its own corpus: one pass to learn the character frequencies, one to emit the + // terms the rule keeps. + printf("%9s %10s %11s %10s %12s %12s %11s %9s\n", "cut-off", "terms", "entries", "per row", + "dict B", "posting B", "index B", "idx/raw"); + for (const double share : {0.02, 0.01, 0.005, 0.0015}) { + const auto threshold = static_cast(corpus.doc_count * share); + std::set common; + for (const writer::TermPostings& t : corpus.terms) { + if (t.docids.size() > threshold) { + common.insert(t.term); + } + } + const Corpus shaped = build_corpus(lines, scheme, mode, &common); + const Built b = build_at(shaped, scheme, threshold); + printf("%8.2f%% %10zu %11llu %10.2f %12llu %12llu %11llu %9.3f\n", share * 100.0, + shaped.terms.size(), static_cast(shaped.posting_entries), + static_cast(shaped.posting_entries) / shaped.doc_count, + static_cast(b.dict_bytes), + static_cast(b.posting_bytes), + static_cast(b.index_bytes), + static_cast(b.index_bytes) / static_cast(shaped.raw_bytes)); + } + return; + } + printf("\ncorpus: %s, %u rows, %.2f MB raw, %zu terms, %llu posting entries " + "(%.2f per row)\n", + path, corpus.doc_count, corpus.raw_bytes / 1e6, corpus.terms.size(), + static_cast(corpus.posting_entries), + static_cast(corpus.posting_entries) / corpus.doc_count); + + // Cut-offs as a share of the segment, from "keep everything" down past the line the + // writer currently draws (0.15%). + const double shares[] = {0.0, 0.50, 0.20, 0.10, 0.05, 0.02, 0.01, 0.005, 0.0015}; + printf("%9s %10s %11s %10s %12s %12s %11s %9s\n", "cut-off", "terms cut", "entries cut", + "entries%", "dict B", "posting B", "index B", "idx/raw"); + for (const double share : shares) { + const uint32_t threshold = + share == 0.0 ? 0 : static_cast(corpus.doc_count * share); + const Built b = build_at(corpus, scheme, threshold); + printf("%8.2f%% %10llu %11llu %9.1f%% %12llu %12llu %11llu %9.3f\n", share * 100.0, + static_cast(b.terms_dropped), + static_cast(b.entries_dropped), + 100.0 * static_cast(b.entries_dropped) / + static_cast(corpus.posting_entries), + static_cast(b.dict_bytes), + static_cast(b.posting_bytes), + static_cast(b.index_bytes), + static_cast(b.index_bytes) / static_cast(corpus.raw_bytes)); + EXPECT_GT(b.index_bytes, 0U); + } +} + +} // namespace doris::snii diff --git a/be/test/storage/index/snii/format/snii_proto_field_numbers_test.cpp b/be/test/storage/index/snii/format/snii_proto_field_numbers_test.cpp index 234bccf3df0025..5a8c1c7e012203 100644 --- a/be/test/storage/index/snii/format/snii_proto_field_numbers_test.cpp +++ b/be/test/storage/index/snii/format/snii_proto_field_numbers_test.cpp @@ -80,7 +80,9 @@ TEST(SniiProtoFieldNumbers, CoreMetadataMatchesTheShippedLayout) { {"stats", 2}, {"section_refs", 3}, {"legacy_common_grams", 4}, - {"legacy_common_grams_posting_policy", 5}}); + {"legacy_common_grams_posting_policy", 5}, + {"gram_scheme", 6}, + {"high_df_terms", 7}}); } } // namespace doris::snii diff --git a/be/test/storage/index/snii/writer/gram_density_calibration_test.cpp b/be/test/storage/index/snii/writer/gram_density_calibration_test.cpp new file mode 100644 index 00000000000000..6e2050c3a082b7 --- /dev/null +++ b/be/test/storage/index/snii/writer/gram_density_calibration_test.cpp @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// The write path solving its own density. +// +// Two properties have to hold together, and they pull against each other. Every row of a +// segment must be cut at ONE rate, because the query side reconstructs a segment's grams from +// the single rate its metadata records -- so the rate has to be known before the first row is +// cut, which is why the first rows are held back. And no row may be lost to that: a segment +// shorter than the sample never fills it, and its rows still have to be indexed. + +#include + +#include +#include +#include + +#include "common/config.h" +#include "gen_cpp/AgentService_types.h" +#include "runtime/exec_env.h" +#include "runtime/index_policy/index_policy_mgr.h" +#include "storage/index/inverted/gram/gram_density.h" +#include "storage/index/snii/snii_index_writer.h" +#include "storage/tablet/tablet_schema.h" +#include "util/defer_op.h" +#include "util/slice.h" + +namespace doris::segment_v2 { +namespace { + +// A gram-family analyzer at a deliberately dense configured rate, so a solved rate is +// distinguishable from the configured one. +class ScopedGramPolicies { +public: + ScopedGramPolicies() { + auto* exec_env = ExecEnv::GetInstance(); + _previous = exec_env->index_policy_mgr(); + exec_env->_index_policy_mgr = &_manager; + + TIndexPolicy tokenizer; + tokenizer.id = 9401; + tokenizer.name = "dens_cal_tokenizer"; + tokenizer.type = TIndexPolicyType::TOKENIZER; + tokenizer.properties["type"] = "ngram"; + tokenizer.properties["mode"] = "sparse"; + tokenizer.properties["min_gram"] = "3"; + tokenizer.properties["max_gram"] = "4"; + tokenizer.properties["density"] = "0.500"; + + TIndexPolicy analyzer; + analyzer.id = 9402; + analyzer.name = "dens_cal_analyzer"; + analyzer.type = TIndexPolicyType::ANALYZER; + analyzer.properties["tokenizer"] = "dens_cal_tokenizer"; + _manager.apply_policy_changes({tokenizer, analyzer}, {}); + } + ~ScopedGramPolicies() { ExecEnv::GetInstance()->_index_policy_mgr = _previous; } + +private: + IndexPolicyMgr _manager; + IndexPolicyMgr* _previous = nullptr; +}; + +TabletIndex make_index_meta() { + TabletIndex meta; + TabletIndexPB pb; + pb.set_index_id(4401); + pb.set_index_name("idx_dens"); + pb.set_index_type(IndexType::INVERTED); + (*pb.mutable_properties())["analyzer"] = "dens_cal_analyzer"; + (*pb.mutable_properties())["support_phrase"] = "false"; + meta.init_from_pb(pb); + return meta; +} + +// Rows long enough to carry windows of the promised length, and varied so their byte pairs do +// not collapse to a handful. +std::vector rows(size_t n) { + std::vector out; + out.reserve(n); + for (size_t i = 0; i < n; ++i) { + out.push_back("rpc error: code = Unavailable desc = transport closing conn " + + std::to_string(i * 7919)); + } + return out; +} + +std::vector slices_of(const std::vector& v) { + std::vector s; + s.reserve(v.size()); + for (const std::string& r : v) { + s.emplace_back(r.data(), r.size()); + } + return s; +} + +size_t term_count(SniiIndexColumnWriter* writer) { + return writer->term_buffer_for_test()->finalize_sorted().size(); +} + +} // namespace + +// Held back, then all of them: nothing written before the rate is settled, and everything +// written once it is. +TEST(GramDensityCalibrationTest, RowsAreHeldBackUntilTheRateIsSolvedAndThenAllIndexed) { + ScopedGramPolicies policies; + const bool saved = config::enable_gram_index_adaptive_density; + const int64_t saved_budget = config::gram_index_density_sample_bytes; + config::enable_gram_index_adaptive_density = true; + config::gram_index_density_sample_bytes = 4096; + Defer restore([&] { + config::enable_gram_index_adaptive_density = saved; + config::gram_index_density_sample_bytes = saved_budget; + }); + + // The term buffer may be drained only once, so the two halves of the property need a + // writer each: one stopped while still calibrating, one carried past the budget. + TabletIndex meta = make_index_meta(); + { + SniiIndexColumnWriter writer(nullptr, &meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_TRUE(writer.init().ok()); + const std::vector first = rows(4); + const std::vector s = slices_of(first); + ASSERT_TRUE(writer.add_values("c", s.data(), s.size()).ok()); + EXPECT_EQ(term_count(&writer), 0U) + << "a row cut before the rate is known could not be cut at the rate the " + "segment will record"; + } + { + SniiIndexColumnWriter writer(nullptr, &meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_TRUE(writer.init().ok()); + const std::vector data = rows(200); + const std::vector s = slices_of(data); + ASSERT_TRUE(writer.add_values("c", s.data(), s.size()).ok()); + EXPECT_GT(term_count(&writer), 0U) << "the held-back rows must be indexed, not dropped"; + } +} + +// The recorded scheme carries the rate the rows were actually cut at, which is what the query +// side rebuilds its grams from. A solved rate that never reached the metadata would make every +// row of the segment unreachable. +TEST(GramDensityCalibrationTest, TheRecordedSchemeCarriesTheSolvedRate) { + ScopedGramPolicies policies; + const bool saved = config::enable_gram_index_adaptive_density; + const int64_t saved_budget = config::gram_index_density_sample_bytes; + config::enable_gram_index_adaptive_density = true; + config::gram_index_density_sample_bytes = 4096; + Defer restore([&] { + config::enable_gram_index_adaptive_density = saved; + config::gram_index_density_sample_bytes = saved_budget; + }); + + TabletIndex meta = make_index_meta(); + SniiIndexColumnWriter writer(nullptr, &meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_TRUE(writer.init().ok()); + ASSERT_TRUE(writer.gram_scheme_for_test().has_value()); + EXPECT_EQ(writer.gram_scheme_for_test()->density_permille, 500) + << "before any row it is still the configured rate"; + + const std::vector data = rows(200); + const std::vector s = slices_of(data); + ASSERT_TRUE(writer.add_values("c", s.data(), s.size()).ok()); + + EXPECT_NE(writer.gram_scheme_for_test()->density_permille, 500) + << "this corpus does not need a rate of 0.5 to find 12-byte literals; a scheme " + "still holding the configured one means the solve never reached it"; + EXPECT_GE(writer.gram_scheme_for_test()->density_permille, gram::kMinSolvedDensityPermille); + EXPECT_LE(writer.gram_scheme_for_test()->density_permille, gram::kMaxSolvedDensityPermille); +} + +// With the feature off nothing is held back and the configured rate stands, so an index built +// before this existed is reproduced exactly. +TEST(GramDensityCalibrationTest, TheConfiguredRateStandsWhenTheSolveIsOff) { + ScopedGramPolicies policies; + const bool saved = config::enable_gram_index_adaptive_density; + config::enable_gram_index_adaptive_density = false; + Defer restore([&] { config::enable_gram_index_adaptive_density = saved; }); + + TabletIndex meta = make_index_meta(); + SniiIndexColumnWriter writer(nullptr, &meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_TRUE(writer.init().ok()); + const std::vector data = rows(4); + const std::vector s = slices_of(data); + ASSERT_TRUE(writer.add_values("c", s.data(), s.size()).ok()); + + EXPECT_GT(term_count(&writer), 0U) << "nothing is held back when there is nothing to solve"; + EXPECT_EQ(writer.gram_scheme_for_test()->density_permille, 500); +} + +} // namespace doris::segment_v2 diff --git a/be/test/storage/index/snii/writer/high_df_digest_cost_test.cpp b/be/test/storage/index/snii/writer/high_df_digest_cost_test.cpp new file mode 100644 index 00000000000000..42a5cea3f6c195 --- /dev/null +++ b/be/test/storage/index/snii/writer/high_df_digest_cost_test.cpp @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// What does building the high-df digest cost the writer? +// +// The digest earns its keep on the query side -- it lets the cost gate bound a node without +// the remote dictionary reads that were measured at 40 seconds on a cold segment. That is +// only a good trade if writing it is close to free, and "close to free" has to be a number +// rather than an assertion. +// +// The two builds below index the identical corpus with the identical settings and differ in +// exactly one field: whether a gram scheme is present, which is what decides that a digest +// is built at all. Everything else -- postings, dictionary blocks, term hashes, statistics +// -- is byte-for-byte the same work, so the difference in elapsed time is the digest. +// +// The shape of the corpus matters to what is measured. Term frequencies follow a Zipf-like +// curve, as a real vocabulary does, so most terms fall below the digest's floor and cost +// only the one comparison that rejects them, while a small head clears it and pays for a +// bounded-heap insertion. Measuring on a flat corpus would either exercise no heap at all +// or exercise nothing but the heap; neither resembles a real index. + +#include + +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii_query_test_util.h" + +namespace doris::snii::writer { +namespace { + +using snii_test::assert_ok; +using snii_test::make_term; +using snii_test::MemoryFile; +using snii_test::PostingDoc; + +constexpr uint32_t kDocCount = 200000; +constexpr uint32_t kTermCount = 40000; +constexpr int kRounds = 5; + +// A Zipf-like vocabulary over kDocCount documents: term i appears in roughly +// kTermCount / (i + 1) documents, so a few hundred terms clear the digest floor +// (kDocCount / kHighDfDigestDivisor = 100) and the rest sit far below it. +std::vector BuildCorpus() { + std::vector terms; + terms.reserve(kTermCount); + for (uint32_t i = 0; i < kTermCount; i++) { + const uint32_t df = std::max(1, kTermCount / (i + 1)); + const uint32_t stride = std::max(1, kDocCount / df); + std::vector docs; + docs.reserve(df); + for (uint32_t docid = 0; docid < kDocCount && docs.size() < df; docid += stride) { + docs.push_back(PostingDoc {.docid = docid, .positions = {}}); + } + // Zero-padded so the terms are already in lexicographic order. + char name[32]; + snprintf(name, sizeof(name), "term_%06u", i); + terms.push_back(make_term(name, std::move(docs))); + } + return terms; +} + +// One build of the whole index. with_digest selects it by supplying a gram scheme, which is +// the only thing that differs between the two arms. +double BuildOnceMs(const std::vector& corpus, bool with_digest, + uint64_t* digest_entries) { + SniiIndexInput input; + input.index_id = 31; + input.index_suffix = "body"; + input.config = format::IndexConfig::kDocsOnly; + input.doc_count = kDocCount; + input.terms = corpus; + if (with_digest) { + segment_v2::gram::GramScheme scheme; + scheme.mode = segment_v2::gram::GramMode::SPARSE; + scheme.min_len = 3; + scheme.max_len = 4; + scheme.density_permille = 250; + input.gram_scheme = scheme; + } + + MemoryFile file; + SniiCompoundWriter writer(&file); + const auto start = std::chrono::steady_clock::now(); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + const auto end = std::chrono::steady_clock::now(); + + if (digest_entries != nullptr) { + reader::SniiSegmentReader segment; + assert_ok(reader::SniiSegmentReader::open(&file, &segment)); + reader::LogicalIndexReader index; + assert_ok(segment.open_index(31, "body", &index)); + *digest_entries = index.high_df_terms().term_hash.size(); + } + return std::chrono::duration(end - start).count(); +} + +double Median(std::vector values) { + std::ranges::sort(values); + return values[values.size() / 2]; +} + +} // namespace + +// Reports the cost; asserts only that it is not a large fraction of the build, so the case +// records a number without becoming a tripwire for ordinary timing noise. +TEST(SniiHighDfDigestCost, BuildingTheDigestIsANegligibleShareOfTheWrite) { + const std::vector corpus = BuildCorpus(); + + uint64_t entries = 0; + std::vector with_digest; + std::vector without; + for (int round = 0; round < kRounds; round++) { + // Alternate the order so a warming effect cannot favour one arm systematically. + if (round % 2 == 0) { + with_digest.push_back(BuildOnceMs(corpus, true, &entries)); + without.push_back(BuildOnceMs(corpus, false, nullptr)); + } else { + without.push_back(BuildOnceMs(corpus, false, nullptr)); + with_digest.push_back(BuildOnceMs(corpus, true, &entries)); + } + } + + const double on = Median(with_digest); + const double off = Median(without); + const double overhead_pct = 100.0 * (on - off) / off; + // Spread of the two arms, so a reader can see whether the difference above is signal. + const double on_spread = + *std::ranges::max_element(with_digest) - *std::ranges::min_element(with_digest); + const double off_spread = + *std::ranges::max_element(without) - *std::ranges::min_element(without); + + printf("\n%u docs, %u terms, digest holds %llu entries\n", kDocCount, kTermCount, + static_cast(entries)); + printf("build with digest : %8.1f ms (median of %d, spread %.1f ms)\n", on, kRounds, + on_spread); + printf("build without digest : %8.1f ms (median of %d, spread %.1f ms)\n", off, kRounds, + off_spread); + printf("difference : %+8.1f ms (%+.2f%%)\n", on - off, overhead_pct); + printf("per term : %+8.3f us\n", 1000.0 * (on - off) / kTermCount); + // The work itself, which is deterministic and does not depend on the machine: one + // comparison for every term, and a bounded-heap insertion only for those above the + // floor. Reported alongside the timing because it is the number that stays true. + printf("work : %u comparisons + at most %llu heap ops (log2 K <= %.1f)\n", + kTermCount, static_cast(entries), + entries > 0 ? std::log2(static_cast(entries)) : 0.0); + + EXPECT_GT(entries, 0U) << "the corpus must actually populate the digest"; + EXPECT_LT(overhead_pct, 5.0) + << "digest build cost " << overhead_pct << "% of the write; it is meant to be " + << "one comparison per term plus a bounded heap insertion for the few that pass"; +} + +} // namespace doris::snii::writer diff --git a/be/test/storage/index/snii/writer/snii_gram_inherit_test.cpp b/be/test/storage/index/snii/writer/snii_gram_inherit_test.cpp new file mode 100644 index 00000000000000..183403eb480313 --- /dev/null +++ b/be/test/storage/index/snii/writer/snii_gram_inherit_test.cpp @@ -0,0 +1,354 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "core/block/block.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/field.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "gen_cpp/AgentService_types.h" +#include "io/fs/local_file_system.h" +#include "runtime/descriptors.h" +#include "runtime/exec_env.h" +#include "runtime/index_policy/index_policy_mgr.h" +#include "runtime/runtime_state.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/index_query_context.h" +#include "storage/index/inverted/gram/gram_scheme.h" +#include "storage/index/inverted/inverted_index_cache.h" +#include "storage/index/inverted/inverted_index_iterator.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/snii_index_reader.h" +#include "storage/index/snii/snii_index_writer.h" +#include "storage/olap_common.h" +#include "storage/segment/column_reader.h" +#include "storage/tablet/tablet_schema.h" +#include "util/defer_op.h" +#include "util/slice.h" + +namespace doris { +namespace { + +using segment_v2::gram::GramMode; +using segment_v2::gram::GramScheme; + +constexpr const char* kTestDir = "./ut_dir/snii_gram_inherit_test"; +constexpr const char* kSourcePrefix = "./ut_dir/snii_gram_inherit_test/source"; +constexpr const char* kInheritedPrefix = "./ut_dir/snii_gram_inherit_test/inherited"; + +struct InheritedPatternResult { + ColumnPtr row_result; + segment_v2::InvertedIndexResultBitmap candidates; + bool has_exact_result = false; +}; + +class SniiGramInheritTest : public testing::Test { +protected: + void SetUp() override { + auto* exec_env = ExecEnv::GetInstance(); + _previous_policy_mgr = exec_env->index_policy_mgr(); + exec_env->_index_policy_mgr = &_policy_mgr; + _previous_query_cache = exec_env->get_inverted_index_query_cache(); + _query_cache.reset( + segment_v2::InvertedIndexQueryCache::create_global_cache(1024 * 1024, 1)); + exec_env->set_inverted_index_query_cache(_query_cache.get()); + + register_index(6753900, "dense", _dense_scheme, &_dense_index); + register_index(6753910, "sparse", _sparse_scheme, &_sparse_index); + register_index(6753920, "fresh", _fresh_scheme, &_fresh_index); + ASSERT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + ASSERT_TRUE(io::global_local_filesystem()->create_directory(kTestDir).ok()); + + TQueryOptions options; + options.__set_query_type(TQueryType::SELECT); + options.__set_enable_hyperscan_fallback(true); + options.__set_enable_inverted_index_query_cache(false); + options.__set_enable_inverted_index_searcher_cache(false); + _runtime_state.set_query_options(options); + _query_context = std::make_shared(); + _query_context->runtime_state = &_runtime_state; + _query_context->stats = &_stats; + _query_context->io_ctx = &_io_context; + } + + void TearDown() override { + auto* exec_env = ExecEnv::GetInstance(); + exec_env->set_inverted_index_query_cache(_previous_query_cache); + _query_cache.reset(); + exec_env->_index_policy_mgr = _previous_policy_mgr; + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + } + + void register_index(int64_t index_id, const std::string& label, const GramScheme& scheme, + TabletIndex* index) { + TIndexPolicy tokenizer; + tokenizer.id = index_id + 1; + tokenizer.name = "gram_inherit_" + label + "_tokenizer"; + tokenizer.type = TIndexPolicyType::TOKENIZER; + tokenizer.properties = scheme.to_properties(); + tokenizer.properties["type"] = "ngram"; + TIndexPolicy analyzer; + analyzer.id = index_id + 2; + analyzer.name = "gram_inherit_" + label + "_analyzer"; + analyzer.type = TIndexPolicyType::ANALYZER; + analyzer.properties = {{"tokenizer", tokenizer.name}}; + _policy_mgr.apply_policy_changes({tokenizer, analyzer}, {}); + + TabletIndexPB index_pb; + index_pb.set_index_type(IndexType::INVERTED); + index_pb.set_index_id(index_id); + index_pb.set_index_name("gram_inherit_" + label + "_index"); + index_pb.add_col_unique_id(0); + index_pb.mutable_properties()->insert({"analyzer", analyzer.name}); + index_pb.mutable_properties()->insert({"support_phrase", "false"}); + index->init_from_pb(index_pb); + } + + Status append_index(segment_v2::IndexFileWriter* file_writer, const TabletIndex& index) { + segment_v2::SniiIndexColumnWriter writer(file_writer, &index, + FieldType::OLAP_FIELD_TYPE_VARCHAR); + RETURN_IF_ERROR(writer.init()); + std::vector slices; + for (const auto& value : _values) { + slices.emplace_back(value); + } + RETURN_IF_ERROR(writer.add_values("p", slices.data(), slices.size())); + return writer.finish(); + } + + Status write_source() { + io::FileWriterPtr file_writer; + RETURN_IF_ERROR(io::global_local_filesystem()->create_file( + std::string(kSourcePrefix) + ".idx", &file_writer)); + segment_v2::IndexFileWriter container( + io::global_local_filesystem(), kSourcePrefix, "gram_inherit_source", 0, + InvertedIndexStorageFormatPB::SNII, std::move(file_writer), true, 6753900); + RETURN_IF_ERROR(append_index(&container, _dense_index)); + RETURN_IF_ERROR(append_index(&container, _sparse_index)); + RETURN_IF_ERROR(container.begin_close()); + return container.finish_close(); + } + + Status inherit_and_append() { + // These are the same public entry points BUILD INDEX uses. The source reader and + // snapshot are local: no source file handle can survive this method's return. + auto source = std::make_shared( + io::global_local_filesystem(), kSourcePrefix, InvertedIndexStorageFormatPB::SNII); + RETURN_IF_ERROR(source->init()); + std::vector keep; + for (const auto* index : {&_dense_index, &_sparse_index}) { + keep.push_back({.index_id = static_cast(index->index_id()), + .index_suffix = index->get_index_suffix()}); + } + snii::reader::SniiRewriteSnapshot snapshot; + RETURN_IF_ERROR(source->prepare_snii_rewrite_snapshot(keep, _values.size(), &snapshot)); + io::FileWriterPtr file_writer; + RETURN_IF_ERROR(io::global_local_filesystem()->create_file( + std::string(kInheritedPrefix) + ".idx", &file_writer)); + segment_v2::IndexFileWriter container( + io::global_local_filesystem(), kInheritedPrefix, "gram_inherit_output", 0, + InvertedIndexStorageFormatPB::SNII, std::move(file_writer), true, 6753900); + RETURN_IF_ERROR(container.inherit_snii(snapshot, source->snii_io_reader())); + RETURN_IF_ERROR(append_index(&container, _fresh_index)); + RETURN_IF_ERROR(container.begin_close()); + return container.finish_close(); + } + + Status evaluate_pattern(const std::shared_ptr& file_reader, + const TabletIndex& index, const std::string& function_name, + const std::string& pattern, InheritedPatternResult* result) { + auto reader = segment_v2::SniiIndexReader::create_shared( + &index, file_reader, segment_v2::InvertedIndexReaderType::FULLTEXT, _values.size(), + /*column_is_array=*/false); + std::vector> iterators(1); + RETURN_IF_ERROR(reader->new_iterator(iterators.data())); + iterators[0]->set_context(_query_context); + + const DataTypePtr string_type = std::make_shared(); + TFunction function; + TFunctionName name; + name.__set_function_name(function_name); + function.__set_name(name); + function.__set_binary_type(TFunctionBinaryType::BUILTIN); + function.__set_arg_types({string_type->to_thrift(), string_type->to_thrift()}); + function.__set_ret_type(DataTypeUInt8().to_thrift()); + function.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::FUNCTION_CALL); + node.__set_type(DataTypeUInt8().to_thrift()); + node.__set_fn(function); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto expression = VectorizedFnCall::create_shared(node); + expression->add_child(VSlotRef::create_shared(0, 0, 0, string_type, "p")); + expression->add_child( + VLiteral::create_shared(string_type, Field::create_field(pattern))); + + std::vector storage_types {{"p", string_type}}; + std::unordered_map> index_status; + segment_v2::ColumnIteratorOptions column_options; + auto index_context = std::make_shared( + iterators, storage_types, index_status, nullptr, nullptr, column_options); + VExprContext context(expression); + context.set_index_context(index_context); + RETURN_IF_ERROR(context.prepare(&_runtime_state, RowDescriptor {})); + RETURN_IF_ERROR(context.open(&_runtime_state)); + // SegmentIterator names the conjunct root as the consumer; without that the gram + // push-down knows its result would be dropped and skips the index read altogether. + context.set_approx_index_result_consumer(expression.get()); + auto column = ColumnString::create(); + for (const auto& value : _values) { + column->insert_data(value.data(), value.size()); + } + Block block; + block.insert({std::move(column), string_type, "p"}); + RETURN_IF_ERROR(context.execute(&block, result->row_result)); + RETURN_IF_ERROR(context.evaluate_inverted_index(_values.size())); + result->has_exact_result = index_context->has_index_result_for_expr(expression.get()); + if (const auto* candidates = + index_context->get_approx_index_result_for_expr(expression.get())) { + result->candidates = *candidates; + } + return Status::OK(); + } + + void check_pattern_result(const InheritedPatternResult& result, + const roaring::Roaring& expected) { + ASSERT_TRUE(result.row_result->get_bool(0)); + ASSERT_FALSE(result.row_result->get_bool(1)); + ASSERT_FALSE(result.row_result->get_bool(2)); + EXPECT_FALSE(result.has_exact_result); + // Missing metadata would fall back to a scalar scan. That is safe for queries, + // but is NOT a passing inheritance test: both schemes must still prune. + ASSERT_FALSE(result.candidates.is_empty()); + EXPECT_TRUE(result.candidates.approximate()); + EXPECT_TRUE(*result.candidates.get_data_bitmap() == expected) + << result.candidates.get_data_bitmap()->toString(); + EXPECT_TRUE(result.candidates.get_data_bitmap()->contains(0)); + EXPECT_FALSE(result.candidates.get_data_bitmap()->contains(2)); + } + + void check_reopened_index(const char* prefix, const TabletIndex& index, + const GramScheme& expected_scheme, + const std::vector& expected_candidates) { + SCOPED_TRACE(prefix); + SCOPED_TRACE(index.index_id()); + auto file_reader = std::make_shared( + io::global_local_filesystem(), prefix, InvertedIndexStorageFormatPB::SNII); + auto status = file_reader->init(); + ASSERT_TRUE(status.ok()) << status; + auto logical_reader = file_reader->open_snii_index(&index); + ASSERT_TRUE(logical_reader.has_value()) << logical_reader.error(); + const auto& actual_scheme = (*logical_reader)->gram_scheme(); + ASSERT_TRUE(actual_scheme.has_value()); + // Equality includes mode, min/max length, density, stop DF, case and hash version. + EXPECT_EQ(expected_scheme.to_properties(), actual_scheme->to_properties()); + + roaring::Roaring expected; + for (uint32_t row : expected_candidates) { + expected.add(row); + } + for (const auto& [name, pattern] : std::vector> { + {"like", "%abcdefgh%"}, {"regexp", "abcdefgh"}}) { + SCOPED_TRACE(name); + InheritedPatternResult result; + status = evaluate_pattern(file_reader, index, name, pattern, &result); + ASSERT_TRUE(status.ok()) << status; + ASSERT_NO_FATAL_FAILURE(check_pattern_result(result, expected)); + } + } + + // Row 1 contains every dense3 gram of "abcdefgh", but no four-byte gram of it. + // It must therefore remain a dense false positive and be pruned by the sparse4 index. + const std::vector _values {"abcdefgh", "abc!bcd!cde!def!efg!fgh", "unrelated"}; + const GramScheme _dense_scheme {.mode = GramMode::DENSE, + .min_len = 3, + .max_len = 7, + .density_permille = 625, + .lower_case = false, + .hash_version = 1}; + const GramScheme _sparse_scheme {.mode = GramMode::SPARSE, + .min_len = 4, + .max_len = 8, + .density_permille = 1000, + .lower_case = false, + .hash_version = 1}; + const GramScheme _fresh_scheme {.mode = GramMode::DENSE, + .min_len = 5, + .max_len = 9, + .density_permille = 500, + .lower_case = false, + .hash_version = 1}; + IndexPolicyMgr _policy_mgr; + IndexPolicyMgr* _previous_policy_mgr = nullptr; + segment_v2::InvertedIndexQueryCache* _previous_query_cache = nullptr; + std::unique_ptr _query_cache; + TabletIndex _dense_index; + TabletIndex _sparse_index; + TabletIndex _fresh_index; + RuntimeState _runtime_state; + OlapReaderStatistics _stats; + io::IOContext _io_context; + segment_v2::IndexQueryContextPtr _query_context; +}; + +TEST_F(SniiGramInheritTest, ReopenedContainerKeepsDistinctSchemesAndPatternCandidates) { + // This is about a container carrying two schemes apart, not about how either was arrived + // at, so it pins the density rather than letting the write path solve one -- a solved rate + // would differ from the scheme the test wrote and constructed its expectation from. + const bool saved_adaptive = doris::config::enable_gram_index_adaptive_density; + doris::config::enable_gram_index_adaptive_density = false; + Defer restore([&] { doris::config::enable_gram_index_adaptive_density = saved_adaptive; }); + auto status = write_source(); + ASSERT_TRUE(status.ok()) << status; + ASSERT_NO_FATAL_FAILURE( + check_reopened_index(kSourcePrefix, _dense_index, _dense_scheme, {0, 1})); + ASSERT_NO_FATAL_FAILURE( + check_reopened_index(kSourcePrefix, _sparse_index, _sparse_scheme, {0})); + + status = inherit_and_append(); + ASSERT_TRUE(status.ok()) << status; + // All source readers have been destroyed. Removing the source additionally ensures that + // reopening the inherited container cannot accidentally consult the original file. + status = io::global_local_filesystem()->delete_file(std::string(kSourcePrefix) + ".idx"); + ASSERT_TRUE(status.ok()) << status; + ASSERT_NO_FATAL_FAILURE( + check_reopened_index(kInheritedPrefix, _dense_index, _dense_scheme, {0, 1})); + ASSERT_NO_FATAL_FAILURE( + check_reopened_index(kInheritedPrefix, _sparse_index, _sparse_scheme, {0})); + ASSERT_NO_FATAL_FAILURE( + check_reopened_index(kInheritedPrefix, _fresh_index, _fresh_scheme, {0})); +} + +} // namespace +} // namespace doris diff --git a/be/test/storage/index/snii/writer/stop_gram_test.cpp b/be/test/storage/index/snii/writer/stop_gram_test.cpp new file mode 100644 index 00000000000000..6dd76e7ce5a134 --- /dev/null +++ b/be/test/storage/index/snii/writer/stop_gram_test.cpp @@ -0,0 +1,329 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// stop-gram on the write side: terms above the df threshold keep their dictionary entry +// and lose their posting list. +// +// The measured motivation, on 200k rows of real log text at the default gram scheme: the +// 855 terms above the query-side cost gate's threshold are 0.7% of the vocabulary and hold +// 88.6% of the posting entries, and the gate refuses to read every one of them. This suite +// pins the three things that has to mean in the file: +// +// * the entry survives, carrying its term key and its real df, so a reader can tell +// "matches everything" from "absent", which mean opposite things; +// * the posting bytes are actually gone -- the saving is the entire point, and an +// implementation that wrote them and merely flagged the entry would pass every +// behavioural assertion while saving nothing; +// * the df is exact, not the count of whatever prefix the writer happened to look at +// before deciding, because the reader's cost gate reads it. + +#include + +#include +#include +#include +#include + +#include "storage/index/snii/format/bsbf.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii_query_test_util.h" + +namespace doris::snii::writer { +namespace { + +using snii_test::assert_ok; +using snii_test::make_term; +using snii_test::MemoryFile; +using snii_test::PostingDoc; + +constexpr uint32_t kDocCount = 4000; + +// A term appearing in every `stride`-th document, so its df is exactly known and can be +// placed either side of a threshold on purpose. +std::vector EveryNth(uint32_t stride) { + std::vector docs; + for (uint32_t docid = 0; docid < kDocCount; docid += stride) { + docs.push_back(PostingDoc {.docid = docid, .positions = {}}); + } + return docs; +} + +struct Segment { + MemoryFile file; + reader::SniiSegmentReader segment; + reader::LogicalIndexReader index; +}; + +// Builds a docs-only GRAM-FAMILY segment holding the given terms, with stop-gram at +// `threshold`. The gram scheme is not decoration: both stop-gram and the high-df digest are +// gated on it, because a match-all term is only sound where the candidates get re-checked +// and the digest is only ever read by the gram query's cost gate. +void Build(Segment* out, const std::vector>& term_strides, + uint32_t threshold) { + SniiIndexInput input; + input.index_id = 11; + input.index_suffix = "body"; + input.config = format::IndexConfig::kDocsOnly; + input.doc_count = kDocCount; + input.stop_gram_df_threshold = threshold; + segment_v2::gram::GramScheme scheme; + scheme.mode = segment_v2::gram::GramMode::SPARSE; + scheme.min_len = 3; + scheme.max_len = 4; + scheme.density_permille = 250; + input.gram_scheme = scheme; + for (const auto& [term, stride] : term_strides) { + input.terms.push_back(make_term(term, EveryNth(stride))); + } + SniiCompoundWriter writer(&out->file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + assert_ok(reader::SniiSegmentReader::open(&out->file, &out->segment)); + assert_ok(out->segment.open_index(11, "body", &out->index)); +} + +struct Looked { + bool found = false; + format::DictEntry entry; +}; + +Looked Lookup(reader::LogicalIndexReader* index, const std::string& term) { + Looked l; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(index->lookup(term, &l.found, &l.entry, &frq_base, &prx_base)); + return l; +} + +} // namespace + +// The threshold splits the vocabulary: below it a term keeps its posting, above it the +// entry stays and the posting goes. +TEST(SniiStopGram, DropsPostingsOnlyAboveTheThreshold) { + // Strides 2 and 4 give df 2000 and 1000; strides 100 and 400 give df 40 and 10. + Segment seg; + Build(&seg, {{"common_a", 2}, {"common_b", 4}, {"rare_a", 100}, {"rare_b", 400}}, + /*threshold=*/500); + + const Looked common_a = Lookup(&seg.index, "common_a"); + ASSERT_TRUE(common_a.found) << "a dropped posting must leave the term in the dictionary"; + EXPECT_TRUE(common_a.entry.posting_dropped); + EXPECT_EQ(common_a.entry.df, 2000U) << "the df must be the real one, not a scan prefix"; + + const Looked common_b = Lookup(&seg.index, "common_b"); + ASSERT_TRUE(common_b.found); + EXPECT_TRUE(common_b.entry.posting_dropped); + EXPECT_EQ(common_b.entry.df, 1000U); + + const Looked rare_a = Lookup(&seg.index, "rare_a"); + ASSERT_TRUE(rare_a.found); + EXPECT_FALSE(rare_a.entry.posting_dropped) << "a term inside the budget keeps its posting"; + EXPECT_EQ(rare_a.entry.df, 40U); + + const Looked rare_b = Lookup(&seg.index, "rare_b"); + ASSERT_TRUE(rare_b.found); + EXPECT_FALSE(rare_b.entry.posting_dropped); + EXPECT_EQ(rare_b.entry.df, 10U); + + // A term that was never indexed is still absent -- the opposite of dropped. + const Looked missing = Lookup(&seg.index, "never_indexed"); + EXPECT_FALSE(missing.found); +} + +// The saving has to be real. Same corpus, same everything, threshold off versus on. +TEST(SniiStopGram, ActuallyRemovesThePostingBytes) { + const std::vector> corpus = { + {"common_a", 2}, {"common_b", 4}, {"rare_a", 100}, {"rare_b", 400}}; + + Segment off; + Build(&off, corpus, /*threshold=*/0); + Segment on; + Build(&on, corpus, /*threshold=*/500); + + EXPECT_LT(on.file.data().size(), off.file.data().size()) + << "dropping the two dense postings must shrink the segment"; + // The two dense terms hold the great majority of the postings here (3000 of 3050 + // entries), so the drop has to be large rather than incidental. + EXPECT_LT(on.file.data().size(), off.file.data().size() * 2 / 3); +} + +// 0 keeps every posting: an index built with the feature off must be byte-identical to one +// built by a version that never had it. +TEST(SniiStopGram, ZeroThresholdDropsNothing) { + Segment seg; + Build(&seg, {{"common_a", 2}, {"rare_a", 100}}, /*threshold=*/0); + EXPECT_FALSE(Lookup(&seg.index, "common_a").entry.posting_dropped); + EXPECT_FALSE(Lookup(&seg.index, "rare_a").entry.posting_dropped); +} + +// A term exactly at the threshold is kept: the rule is "above", so the boundary term is +// still readable and the reader's gate -- which uses the same comparison -- agrees. +TEST(SniiStopGram, TheTermExactlyAtTheThresholdIsKept) { + Segment seg; + // stride 4 over 4000 docs is df 1000 exactly. + Build(&seg, {{"boundary", 4}}, /*threshold=*/1000); + const Looked boundary = Lookup(&seg.index, "boundary"); + ASSERT_TRUE(boundary.found); + EXPECT_FALSE(boundary.entry.posting_dropped) << "df == threshold is not above it"; + EXPECT_EQ(boundary.entry.df, 1000U); +} + +// ---- the high-df digest, which the query side uses to skip reading df at all ---- + +TEST(SniiStopGram, HighDfDigestRecordsTheCommonTermsAndBoundsTheRest) { + // 4000 docs puts the digest floor at 4000 / kHighDfDigestDivisor = 2, so terms at + // stride 2 and 4 (df 2000 and 1000) are well above it and the two rare ones are not. + Segment seg; + Build(&seg, {{"common_a", 2}, {"common_b", 4}, {"rare_a", 100}, {"rare_b", 400}}, + /*threshold=*/0); + + const auto& digest = seg.index.high_df_terms(); + ASSERT_FALSE(digest.empty()) << "terms this common must reach the digest"; + + bool exact = false; + EXPECT_EQ(digest.df_upper_bound(format::bsbf_hash("common_a"), &exact), 2000U); + EXPECT_TRUE(exact) << "a term in the digest yields its real df, not a bound"; + EXPECT_EQ(digest.df_upper_bound(format::bsbf_hash("common_b"), &exact), 1000U); + EXPECT_TRUE(exact); + + // A term the digest omits still gets a usable answer -- an upper bound rather than an + // exact df. That is what lets the query side decide a node is worth reading without + // touching the dictionary. + const uint32_t bound = digest.df_upper_bound(format::bsbf_hash("never_indexed"), &exact); + EXPECT_FALSE(exact); + EXPECT_EQ(bound, digest.df_ceiling); + EXPECT_LT(bound, 1000U) << "the ceiling must sit below the terms the digest kept"; +} + +// The digest has to stay proportional to the index it describes. On a segment this small +// the df floor is 2 documents, so most of the vocabulary clears it and only the cap keeps +// the digest from dwarfing the index -- the failure this pins is a 45 KB digest attached to +// a 96 KB index, which is what an absolute cap alone produced. +TEST(SniiStopGram, HighDfDigestStaysProportionalToTheVocabulary) { + std::vector> corpus; + // 300 distinct terms, every one of them appearing in at least two documents and so + // above this segment's floor. + for (uint32_t i = 0; i < 300; i++) { + corpus.emplace_back("term_" + std::to_string(i), 2 + (i % 50)); + } + Segment seg; + Build(&seg, corpus, /*threshold=*/0); + + const auto& digest = seg.index.high_df_terms(); + const uint64_t terms = seg.index.stats().term_count; + ASSERT_GT(terms, 0U); + const size_t allowed = std::min( + format::kMaxHighDfDigestTerms, + std::max(format::kMinHighDfDigestTerms, + static_cast(terms) / format::kHighDfDigestVocabularyShare)); + EXPECT_LE(digest.term_hash.size(), allowed) + << "digest of " << digest.term_hash.size() << " entries over " << terms << " terms"; + // Truncating must not cost the reader its bound: everything dropped is still covered. + EXPECT_GT(digest.df_ceiling, 0U) << "a truncated digest must still bound what it omits"; +} + +// An index no gram query can reach must not carry the digest at all. It would be pure loss: +// bytes in the metadata and cycles on every term, for a structure only the gram cost gate +// consults. Segments of such indexes must stay byte-identical to what an earlier build +// produced. +TEST(SniiStopGram, NonGramIndexesCarryNoDigest) { + SniiIndexInput input; + input.index_id = 11; + input.index_suffix = "body"; + input.config = format::IndexConfig::kDocsOnly; + input.doc_count = kDocCount; + // No gram_scheme: an ordinary full-text index. + input.terms.push_back(make_term("common_a", EveryNth(2))); + input.terms.push_back(make_term("rare_a", EveryNth(100))); + + MemoryFile file; + SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + reader::SniiSegmentReader segment; + assert_ok(reader::SniiSegmentReader::open(&file, &segment)); + reader::LogicalIndexReader index; + assert_ok(segment.open_index(11, "body", &index)); + + const auto& digest = index.high_df_terms(); + EXPECT_TRUE(digest.empty()) << "a non-gram index must build no digest"; + EXPECT_EQ(digest.df_ceiling, 0U) + << "and no ceiling either, so the reader knows there is no bound rather than " + "mistaking zero for one"; +} + +// Same standing-down at the small end, and for the same reason the reader and stop-gram +// stand down there: a floor clamped to 1 makes "common" mean "occurs twice". +TEST(SniiStopGram, SegmentsBelowTheDivisorCarryNoDigest) { + SniiIndexInput input; + input.index_id = 11; + input.index_suffix = "body"; + input.config = format::IndexConfig::kDocsOnly; + input.doc_count = format::kHighDfDigestDivisor - 1; + segment_v2::gram::GramScheme scheme; + scheme.mode = segment_v2::gram::GramMode::SPARSE; + scheme.min_len = 3; + scheme.max_len = 4; + scheme.density_permille = 250; + input.gram_scheme = scheme; + std::vector docs; + for (uint32_t d = 0; d < input.doc_count; d += 2) { + docs.push_back(PostingDoc {.docid = d, .positions = {}}); + } + input.terms.push_back(make_term("everywhere", docs)); + + MemoryFile file; + SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + reader::SniiSegmentReader segment; + assert_ok(reader::SniiSegmentReader::open(&file, &segment)); + reader::LogicalIndexReader index; + assert_ok(segment.open_index(11, "body", &index)); + + EXPECT_TRUE(index.high_df_terms().empty()); + EXPECT_EQ(index.high_df_terms().df_ceiling, 0U); +} + +// The digest is a binary search over ascending hashes; unsorted content would return wrong +// bounds silently rather than fail, so the writer has to emit it sorted. +TEST(SniiStopGram, HighDfDigestIsAscendingByHash) { + Segment seg; + Build(&seg, {{"a_common", 2}, {"b_common", 3}, {"c_common", 4}, {"d_common", 5}}, + /*threshold=*/0); + const auto& digest = seg.index.high_df_terms(); + ASSERT_GE(digest.term_hash.size(), 2U); + EXPECT_TRUE(std::ranges::is_sorted(digest.term_hash)); + EXPECT_EQ(digest.term_hash.size(), digest.df.size()); +} + +// A df far above the first fill still comes out exact: the writer drains the source rather +// than reporting the prefix it examined before deciding. +TEST(SniiStopGram, ReportsAnExactDfForAVeryCommonTerm) { + Segment seg; + Build(&seg, {{"everywhere", 1}}, /*threshold=*/10); + const Looked everywhere = Lookup(&seg.index, "everywhere"); + ASSERT_TRUE(everywhere.found); + EXPECT_TRUE(everywhere.entry.posting_dropped); + EXPECT_EQ(everywhere.entry.df, kDocCount); +} + +} // namespace doris::snii::writer diff --git a/be/test/storage/index/snii_writer_test.cpp b/be/test/storage/index/snii_writer_test.cpp index bae359c434d881..1d3dc507c6dc13 100644 --- a/be/test/storage/index/snii_writer_test.cpp +++ b/be/test/storage/index/snii_writer_test.cpp @@ -42,6 +42,9 @@ #include "runtime/index_policy/index_policy_mgr.h" #include "storage/index/inverted/analyzer/analyzer.h" #include "storage/index/inverted/analyzer/custom_analyzer.h" +#include "storage/index/inverted/gram/gram_extractor.h" +#include "storage/index/inverted/gram/gram_family.h" +#include "storage/index/inverted/tokenizer/ngram/gram_tokenizer.h" #include "storage/index/snii/common/slice.h" #include "storage/index/snii/encoding/byte_sink.h" #include "storage/index/snii/format/dict_block.h" @@ -53,14 +56,86 @@ #include "storage/index/snii/query/bm25_scorer.h" #include "storage/index/snii/query/docid_sink.h" #include "storage/index/snii/snii_index_writer.h" +#include "storage/index/snii/writer/logical_index_writer.h" #include "storage/tablet/tablet_schema.h" #include "util/slice.h" +namespace doris::segment_v2 { +uint32_t snii_effective_stop_gram_df_threshold(const snii::writer::SniiIndexInput& input); +} // namespace doris::segment_v2 + namespace { +// The write path holds a segment's first rows back to solve its density from them (see +// gram_density.h). These tests inspect the term buffer directly, without a flush, so they pin +// the density instead: what they are about is how a row is cut at a given rate. +class ScopedConfiguredDensity { +public: + ScopedConfiguredDensity() : _saved(doris::config::enable_gram_index_adaptive_density) { + doris::config::enable_gram_index_adaptive_density = false; + } + ~ScopedConfiguredDensity() { doris::config::enable_gram_index_adaptive_density = _saved; } + +private: + bool _saved; +}; using doris::snii::ByteSink; using doris::snii::Slice; using namespace doris::snii::format; // NOLINT(google-build-using-namespace) +namespace gram = doris::segment_v2::gram; + +// Inject a gram-family (ngram tokenizer + mode=sparse) tokenizer/analyzer policy pair, swapping +// ExecEnv::_index_policy_mgr for a private manager and restoring it afterwards, for the tests +// of gram-family detection and the SNII writer forcing docs-only. +class ScopedGramPolicies { +public: + ScopedGramPolicies() { + auto* exec_env = doris::ExecEnv::GetInstance(); + previous_ = exec_env->index_policy_mgr(); + exec_env->_index_policy_mgr = &manager_; + + // IndexPolicyMgr's _name_to_id is a single namespace across policy types + // (apply_policy_changes deduplicates by name and rejects a later policy outright when its + // name clashes with an existing one, see index_policy_mgr.cpp:86-92). A tokenizer and an + // analyzer therefore cannot share a name, so the tokenizer gets its own + // "gram_sparse_tokenizer" while the analyzer keeps "gram_sparse" (the name the index's + // analyzer property refers to). + doris::TIndexPolicy tokenizer; + tokenizer.id = 9001; + tokenizer.name = "gram_sparse_tokenizer"; + tokenizer.type = doris::TIndexPolicyType::TOKENIZER; + tokenizer.properties["type"] = "ngram"; + tokenizer.properties["mode"] = "sparse"; + + doris::TIndexPolicy analyzer; + analyzer.id = 9002; + analyzer.name = "gram_sparse"; + analyzer.type = doris::TIndexPolicyType::ANALYZER; + analyzer.properties["tokenizer"] = "gram_sparse_tokenizer"; + + // The same gram-family tokenizer, but with a token filter attached: R22 requires such an + // analyzer never to count as gram family (the filter rewrites tokens, so the stored term + // is no longer what the extractor produced). + doris::TIndexPolicy filtered_analyzer; + filtered_analyzer.id = 9003; + filtered_analyzer.name = filtered_analyzer_name(); + filtered_analyzer.type = doris::TIndexPolicyType::ANALYZER; + filtered_analyzer.properties["tokenizer"] = "gram_sparse_tokenizer"; + filtered_analyzer.properties["token_filter"] = "lowercase"; + + manager_.apply_policy_changes({tokenizer, analyzer, filtered_analyzer}, {}); + } + + ~ScopedGramPolicies() { doris::ExecEnv::GetInstance()->_index_policy_mgr = previous_; } + + static std::string filtered_analyzer_name() { return "gram_sparse_filtered"; } + + doris::IndexPolicyMgr& manager() { return manager_; } + +private: + doris::IndexPolicyMgr manager_; + doris::IndexPolicyMgr* previous_ = nullptr; +}; // A fatal assertion inside a helper FUNCTION only aborts the helper; the calling // test keeps running and may dereference state that failed to initialize (this @@ -125,6 +200,21 @@ class ThrowingAnalyzer final : public lucene::analysis::Analyzer { private: std::unique_ptr _reusable; }; +// Generic index_meta construction: push an arbitrary property table into a fresh TabletIndex, so +// the gram-family detection tests can pass custom properties directly instead of hardcoding them +// in the function body like the two dedicated helpers above. +void init_gram_index_meta(doris::TabletIndex* index_meta, int64_t index_id, + const std::map& properties) { + doris::TabletIndexPB index_pb; + index_pb.set_index_type(doris::IndexType::INVERTED); + index_pb.set_index_id(index_id); + index_pb.set_index_name("gram_family_writer"); + index_pb.add_col_unique_id(0); + for (const auto& [key, value] : properties) { + index_pb.mutable_properties()->insert({key, value}); + } + index_meta->init_from_pb(index_pb); +} std::shared_ptr create_failure_analyzer() { return std::make_shared(); @@ -136,6 +226,90 @@ doris::Slice malformed_value_after_valid_token() { return doris::Slice(input); } +// The stop-gram resolver decides whether an index may drop posting lists at all. Each of +// its three conditions is a correctness guard rather than a tuning choice, so each gets +// pinned here: dropping a posting makes a term match every document, which only a +// re-checked (gram) predicate survives, only a docs-only index can afford (there are no +// positions to lose), and only a segment worth the saving should pay for. There is no +// switch: these three conditions are the whole policy. +TEST(SniiWriterTest, EffectiveStopGramThresholdResolver) { + doris::snii::writer::SniiIndexInput input; + input.config = IndexConfig::kDocsOnly; + input.doc_count = 1000000; + + // No gram scheme: an ordinary full-text index has no re-check to make a match-all + // term safe. + EXPECT_EQ(doris::segment_v2::snii_effective_stop_gram_df_threshold(input), 0U); + + doris::segment_v2::gram::GramScheme scheme; + scheme.mode = doris::segment_v2::gram::GramMode::SPARSE; + scheme.min_len = 3; + scheme.max_len = 16; + scheme.density_permille = 250; + input.gram_scheme = scheme; + // Derived, not configured: doc_count / kHighDfDigestDivisor, times the query side's + // budget multiple -- the same line the reader's gate draws. + EXPECT_EQ(doris::segment_v2::snii_effective_stop_gram_df_threshold(input), 1500U) + << "1,000,000 / 2000 * 3"; + + // Positions would be dropped along with the posting, taking phrase capability with + // them, so a positional index is refused even with a gram scheme. + input.config = IndexConfig::kDocsPositions; + EXPECT_EQ(doris::segment_v2::snii_effective_stop_gram_df_threshold(input), 0U); + input.config = IndexConfig::kDocsOnly; + + // The threshold scales with the segment rather than with a constant: a tenth of the + // rows gives a tenth of the line. + input.doc_count = 100000; + EXPECT_EQ(doris::segment_v2::snii_effective_stop_gram_df_threshold(input), 150U); + input.doc_count = 1000000; + + // A segment with fewer rows than the digest's divisor has no meaningful notion of a + // common term, and both sides stand down there: the reader's digest ceiling lands at 0 + // and its gate falls back rather than deriving a budget, so the writer must not drop + // postings either. + input.doc_count = 100; + EXPECT_EQ(doris::segment_v2::snii_effective_stop_gram_df_threshold(input), 0U); + input.doc_count = 1999; + EXPECT_EQ(doris::segment_v2::snii_effective_stop_gram_df_threshold(input), 0U); + input.doc_count = 2000; + EXPECT_EQ(doris::segment_v2::snii_effective_stop_gram_df_threshold(input), 3U) + << "the first segment size at which the line is derived rather than clamped"; +} + +// What the threshold costs, pinned so it cannot be forgotten. +// +// The line is drawn for size: it drops 88.6% of all posting entries on log text and takes +// index/data from 1.097 to 0.218, which is what the index's size target requires. It is far +// below the df of the terms a CJK query is made of -- those columns index one term per code +// point, and an ordinary character sits in a few percent of the rows. Measured on weibo with +// stop-gram enabled, a four-character literal proposes 500,000 candidates for its 18 matching +// rows and eliminates none, against 148 candidates and 499,852 rows eliminated without it. +// ASCII is not exempt either: `/history/images/` matches 0.73% of httplogs and loses its +// filtering the same way. What survives is the needle -- 13 rows in 3,000,000 keeps a 121x +// cold speedup. +// +// The assertion below is the arithmetic that produces that outcome. If someone raises this +// line to protect CJK, this test is where they will see what it was bought with. +TEST(SniiWriterTest, StopGramLineSitsFarBelowOrdinaryCjkCharacterFrequency) { + doris::snii::writer::SniiIndexInput input; + input.config = IndexConfig::kDocsOnly; + input.doc_count = 500000; + doris::segment_v2::gram::GramScheme scheme; + scheme.mode = doris::segment_v2::gram::GramMode::SPARSE; + scheme.min_len = 3; + scheme.max_len = 16; + scheme.density_permille = 250; + input.gram_scheme = scheme; + + const uint32_t threshold = doris::segment_v2::snii_effective_stop_gram_df_threshold(input); + EXPECT_EQ(threshold, 750U) << "500,000 / 2000 * 3, i.e. 0.15% of the segment"; + // A character in 1% of the posts is dropped, and it would still have removed 99% of the + // rows. That is the trade, stated as a number rather than left to be rediscovered. + EXPECT_LT(threshold, 5000U) + << "terms that still filter are dropped; this is deliberate, for size"; +} + // SampledTermIndexReader::heap_bytes(): all-SSO sample terms have no per-string // heap, so the charge is exactly n_blocks * sizeof(std::string) (reserve-exact // backing buffer). @@ -377,6 +551,225 @@ TEST(SniiWriterNorms, WritesNormsFollowSharedNormsPolicy) { doris::config::inverted_index_skip_norms_for_variant = original_skip_norms_for_variant; } +TEST(SniiWriterTest, GramTokenizerForcesDocsOnlyAndIsRecognised) { + ScopedConfiguredDensity fixed_density; + ScopedGramPolicies policies; + + const std::map props {{"analyzer", "gram_sparse"}, + {"support_phrase", "true"}}; // ask for pos + doris::TabletIndex index_meta; + init_gram_index_meta(&index_meta, 9010, props); + + auto scheme = gram::resolve_gram_scheme(index_meta.properties(), &policies.manager()); + ASSERT_TRUE(scheme.has_value()); + EXPECT_EQ(scheme->mode, gram::GramMode::SPARSE); + + doris::segment_v2::SniiIndexColumnWriter writer(nullptr, &index_meta, + doris::FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_OK(writer.init()); + // The gram family forces docs-only and ignores support_phrase. + EXPECT_EQ(writer.config_for_test(), IndexConfig::kDocsOnly); + ASSERT_TRUE(writer.gram_scheme_for_test().has_value()); + EXPECT_TRUE(writer.gram_scheme_for_test().value() == *scheme); + + const std::vector values {doris::Slice("rpc error: code = Unavailable"), + doris::Slice("手机微博")}; + ASSERT_OK(writer.add_values("c", values.data(), values.size())); + auto postings = writer.term_buffer_for_test()->finalize_sorted(); + std::vector terms; + terms.reserve(postings.size()); + for (auto& posting : postings) { + terms.push_back(posting.term); + } + std::ranges::sort(terms); + // Non-ASCII text contributes no terms: only the ASCII segment of the row is indexed. + std::vector expected {" Unavai", "ailable", "cod", "ode = U", "or: co"}; + std::ranges::sort(expected); + EXPECT_EQ(terms, expected); +} + +// Regression guard for the reusable-token-stream lane. The gram family pulls its stream from +// Analyzer::reusableTokenStream, which differs from tokenStream in two ways that both bite here: +// it hands back the SAME cached stream every row, and it does NOT reset it. The writer has to +// reset it itself -- DorisTokenizer::reset() is what promotes the new reader from _in_pending to +// _in, and GramTokenizer::reset() is what re-extracts and rewinds the cursor. Skip that and every +// row after the first is indexed as a repeat of the first, silently. Checking the union of terms +// would not catch it; checking which docid each term landed under does. The three rows below have +// pairwise disjoint gram sets and deliberately differ in length. +TEST(SniiWriterTest, GramRowsStayIndependentAcrossTheReusedTokenStream) { + ScopedConfiguredDensity fixed_density; + ScopedGramPolicies policies; + + const std::map props {{"analyzer", "gram_sparse"}}; + doris::TabletIndex index_meta; + init_gram_index_meta(&index_meta, 9014, props); + + doris::segment_v2::SniiIndexColumnWriter writer(nullptr, &index_meta, + doris::FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_OK(writer.init()); + ASSERT_TRUE(writer.gram_scheme_for_test().has_value()); + + const std::vector values { + doris::Slice("rpc error: code = Unavailable desc = transport is closing"), + doris::Slice("手机微博"), doris::Slice("SELECT COUNT(*) FROM tbl WHERE k = 42")}; + ASSERT_OK(writer.add_values("c", values.data(), values.size())); + + // Ground truth: the same extractor the writer used, driven one row at a time. + gram::GramExtractor extractor(*writer.gram_scheme_for_test()); + std::map> expected; + for (uint32_t docid = 0; docid < values.size(); docid++) { + std::vector grams; + extractor.grams_of_literal(std::string_view(values[docid].data, values[docid].size), + &grams); + for (auto& gram : grams) { + expected[gram].push_back(docid); + } + } + ASSERT_GE(expected.size(), 3U); + + std::map> actual; + for (auto& posting : writer.term_buffer_for_test()->finalize_sorted()) { + actual[posting.term] = posting.docids; + } + EXPECT_EQ(actual, expected); +} + +// The tokenizer's scratch is ordinary heap, so the process-wide MemTrackerLimiter always saw it +// through the allocation hook -- but this writer's own MemoryReporter did not, so the number +// driving its flush/spill decision understated real RSS. A long, highly repetitive row makes the +// wiring checkable: SPARSE collapses it to a handful of distinct grams, so the term buffer (the +// other contributor to the reporter) stays small while the tokenizer's scratch is sized by the +// row. Without _report_gram_buffers_capacity the total sits far below the scratch alone. +TEST(SniiWriterTest, GramTokenizerBuffersReachTheMemoryReporter) { + ScopedConfiguredDensity fixed_density; + ScopedGramPolicies policies; + + const std::map props {{"analyzer", "gram_sparse"}}; + doris::TabletIndex index_meta; + init_gram_index_meta(&index_meta, 9015, props); + + doris::segment_v2::SniiIndexColumnWriter writer(nullptr, &index_meta, + doris::FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_OK(writer.init()); + ASSERT_TRUE(writer.gram_scheme_for_test().has_value()); + + std::string row; + while (row.size() < 256 * 1024) { + row += "2026-09-06 INFO rpc request completed service=Frontend latency_ms=17 "; + } + const doris::Slice value(row.data(), row.size()); + ASSERT_OK(writer.add_values("c", &value, 1)); + + // Ground truth: the same tokenizer on the same row, driven standalone. reset() is what fills + // the buffers, so their capacity is settled by the time it returns. + doris::segment_v2::inverted_index::GramTokenizer tokenizer(*writer.gram_scheme_for_test()); + auto reader = std::make_shared>(); + reader->init(row.data(), static_cast(row.size()), false); + tokenizer.set_reader(reader); + tokenizer.reset(); + const size_t scratch_bytes = tokenizer.reserved_bytes(); + ASSERT_GT(scratch_bytes, 512U * 1024U) << "the row is meant to dwarf the distinct-term set"; + + EXPECT_GE(writer.memory_reporter_for_test()->current_bytes(), + static_cast(scratch_bytes)); +} + +// A non-gram-family index (the built-in parser="english", with no analyzer/normalizer property) +// must resolve to no scheme, whether or not the process's current IndexPolicyMgr is empty +// (resolve_gram_scheme short-circuits to nullopt for both mgr==nullptr and "empty analyzer name", +// without depending on whether the policy manager has been initialized). +TEST(SniiWriterTest, NonGramAnalyzerHasNoScheme) { + const std::map props {{"parser", "english"}}; + doris::TabletIndex index_meta; + init_gram_index_meta(&index_meta, 9011, props); + EXPECT_FALSE(gram::resolve_gram_scheme(index_meta.properties(), + doris::ExecEnv::GetInstance()->index_policy_mgr()) + .has_value()); +} + +// R21 regression guard: built-in analyzer names (standard/english/...) are never registered as +// index policies, so asking the policy manager for one always throws "Policy not found". If +// gram-family detection went through the policy manager, every existing index with +// PROPERTIES("analyzer"="standard") would fail to build -- so this deliberately installs a +// manager holding gram policies only, making sure a built-in name neither enters the manager nor +// changes the previous config conclusion. +TEST(SniiWriterTest, BuiltinAnalyzerNameStillInitialises) { + ScopedGramPolicies policies; // only gram policies in the manager, no "standard"/"english" + const std::vector> cases { + {{"analyzer", "standard"}, {"support_phrase", "true"}}, + {{"parser", "english"}}}; // the legacy parser spelling + int64_t index_id = 9012; + for (const auto& props : cases) { + doris::TabletIndex index_meta; + init_gram_index_meta(&index_meta, index_id++, props); + doris::segment_v2::SniiIndexColumnWriter writer(nullptr, &index_meta, + doris::FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_OK(writer.init()); + EXPECT_FALSE(writer.gram_scheme_for_test().has_value()); + // config is decided purely by support_phrase, independently of gram-family detection. + EXPECT_EQ(writer.config_for_test(), props.contains("support_phrase") + ? IndexConfig::kDocsPositions + : IndexConfig::kDocsOnly); + } +} + +// The negative case: an analyzer name that is neither built-in nor present in the policy manager +// -- a genuine configuration error, which must keep its existing behaviour (an +// INVERTED_INDEX_ANALYZER_ERROR while creating the analyzer) and must not be swallowed by R21's +// short circuit. +TEST(SniiWriterTest, MissingAnalyzerPolicyFailsInit) { + ScopedGramPolicies policies; + const std::map props {{"analyzer", "no_such_policy"}}; + doris::TabletIndex index_meta; + init_gram_index_meta(&index_meta, 9014, props); + doris::segment_v2::SniiIndexColumnWriter writer(nullptr, &index_meta, + doris::FieldType::OLAP_FIELD_TYPE_VARCHAR); + const auto status = writer.init(); + EXPECT_EQ(status.code(), doris::ErrorCode::INVERTED_INDEX_ANALYZER_ERROR) << status; +} + +// R22: once a token filter is attached to a gram-family tokenizer, the stored term is no longer +// GramExtractor.extract(raw column value), so it must be treated as "not gram family" -- neither +// reporting a scheme nor quietly dropping the phrase positions the user asked for. +TEST(SniiWriterTest, TokenFilteredGramAnalyzerIsNotGramFamily) { + ScopedGramPolicies policies; + const std::string analyzer_name = ScopedGramPolicies::filtered_analyzer_name(); + auto provider = policies.manager().get_analyzer_provider_by_name(analyzer_name); + ASSERT_NE(provider, nullptr); + EXPECT_FALSE(provider->gram_scheme().has_value()); + + const std::map props {{"analyzer", analyzer_name}, + {"support_phrase", "true"}}; + doris::TabletIndex index_meta; + init_gram_index_meta(&index_meta, 9015, props); + doris::segment_v2::SniiIndexColumnWriter writer(nullptr, &index_meta, + doris::FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_OK(writer.init()); + EXPECT_FALSE(writer.gram_scheme_for_test().has_value()); + EXPECT_EQ(writer.config_for_test(), IndexConfig::kDocsPositions); // docs-only not forced +} + +// The other half of R22: an index-level char_filter is wrapped around the reader by the writer +// itself and is invisible to the policy provider, so the provider still reports a scheme -- the +// writer has to suppress it. +TEST(SniiWriterTest, IndexLevelCharFilterIsNotGramFamily) { + ScopedGramPolicies policies; + EXPECT_TRUE(policies.manager() + .get_analyzer_provider_by_name("gram_sparse") + ->gram_scheme() + .has_value()); + + const std::map props {{"analyzer", "gram_sparse"}, + {"char_filter_type", "char_replace"}, + {"char_filter_pattern", "._"}}; + doris::TabletIndex index_meta; + init_gram_index_meta(&index_meta, 9016, props); + doris::segment_v2::SniiIndexColumnWriter writer(nullptr, &index_meta, + doris::FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_OK(writer.init()); + EXPECT_FALSE(writer.gram_scheme_for_test().has_value()); +} + TEST(SniiDocIdSinkGrowth, AppendRangeGrowsGeometrically) { std::vector docids; doris::snii::query::VectorDocIdSink sink(docids); diff --git a/build-support/check-header-deps.py b/build-support/check-header-deps.py index 109f4b4d8d5c5f..e50a58f095688c 100755 --- a/build-support/check-header-deps.py +++ b/build-support/check-header-deps.py @@ -495,7 +495,15 @@ "core/field.h": 60, "core/column/column.h": 66, "exprs/function/function.h": 106, - "exec/pipeline/dependency.h": 357, + # +1 (was 357): storage/index/snii/format/core_metadata.h now carries an + # optional gram::GramScheme member (SNII core metadata gram_scheme field, + # regex sparse-gram-index P0 Task 12), reachable via + # inverted_index_reader.h -> inverted_index_cache.h -> + # snii/reader/logical_index_reader.h -> snii/format/core_metadata.h -> + # inverted/gram/gram_scheme.h. gram_scheme.h itself is a leaf (cstdint, + # map, string, common/status.h -- the last already on this hub), so this + # is exactly +1 project header, not a chain. + "exec/pipeline/dependency.h": 358, "pch/pch.h": 8, } diff --git a/gensrc/proto/snii.proto b/gensrc/proto/snii.proto index 1dc6793b1e2798..70b4506441ac12 100644 --- a/gensrc/proto/snii.proto +++ b/gensrc/proto/snii.proto @@ -82,6 +82,42 @@ message SniiCoreMetadataPB { // the contents. Older production 4.1.7-series readers already ignore these fields. optional bytes legacy_common_grams = 4; optional uint32 legacy_common_grams_posting_policy = 5; + // Chunking scheme of a gram-family index (the sparse/dense/auto modes of the ngram + // tokenizer). The writer records the scheme it actually used and the query side compiles + // patterns against it, so the two can never disagree. Absent on every index that is not gram + // family, which leaves the encoded bytes of those segments unchanged. + // Append only -- the numbers of the first 5 fields must not change. + optional SniiGramSchemePB gram_scheme = 6; + // Digest of this index's highest-df terms, so a query can bound its own candidate count + // before reading a single dictionary block. See SniiHighDfTermsPB. + optional SniiHighDfTermsPB high_df_terms = 7; +} + +// The highest-df terms of one index, as (hash, df) pairs, small enough to travel with the +// core metadata and be resident for the life of the segment. +// +// It exists to answer one question without IO: is this query's candidate set already too +// large for the index to be worth reading? The query side's cost gate needs min(df) over an +// AND's terms, or max(df) over an OR's, and reading those df values means one remote +// dictionary round trip per term. On a segment whose cache holds nothing, that round trip +// has been measured at 40 seconds -- paid in full before the gate decides to give up. +// +// A miss is as informative as a hit: the digest holds the top-K terms by df, so any term +// absent from it has df <= df_ceiling. That upper bound is what lets an AND proceed without +// consulting the dictionary at all. +// +// K scales with the index's own vocabulary rather than being configured, and is capped so +// the digest stays small. Real log text puts 88.6% of all posting entries in the top 0.7% +// of terms, so a small K captures nearly every term a gate would reject. +message SniiHighDfTermsPB { + // bsbf_hash of each term, ascending, so a lookup is a binary search. + repeated fixed64 term_hash = 1 [packed = true]; + // Document frequencies, parallel to term_hash. + repeated uint32 df = 2 [packed = true]; + // Every term NOT in this digest has df <= this value (it is the smallest df present). + // Absent digest or an empty one means no bound is available and the reader must fall + // back to reading df from the dictionary. + optional uint32 df_ceiling = 3; } // FIELD NUMBERS ARE ON DISK. The SNII format shipped before this message reached @@ -115,3 +151,22 @@ message SniiSectionRefsPB { optional SniiRegionRefPB norms = 5; } +// Scheme parameters of a gram-family index (the sparse/dense modes of the ngram tokenizer), the +// single source of truth by which downstream components such as GramExtractor and +// RegexGramCompiler split grams. The writer persists the scheme it actually used into every +// gram-family segment, and the query side compiles LIKE/REGEXP patterns against the scheme read +// back from that same segment, so a pattern is always split exactly the way the data was. Mode +// "auto" is a write-side input only: it is resolved to dense or sparse before it is stored here, +// so this message never carries it. +// Unlike the messages above, this one has never shipped: it is introduced together with the +// gram-family index itself, so no released version has ever written a tag of it and the numbering +// is still free to change. Once this lands, the same "field numbers are on disk" rule applies here +// as everywhere else in this file. +message SniiGramSchemePB { + optional uint32 mode = 1; // on-disk encoding of GramMode: 1 = dense, 2 = sparse + optional uint32 min_len = 2; + optional uint32 max_len = 3; + optional uint32 density_permille = 4; + optional bool lower_case = 5; + optional uint32 hash_version = 6; +} From 6f684062c293949a4daab74a11c9e91c5f424481 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Thu, 10 Sep 2026 20:55:19 +0800 Subject: [PATCH 04/36] [feature](index) LIKE/REGEXP push-down through GRAM_BOOLEAN_QUERY with a cost gate and approximate index results Query side of the gram index. - FunctionLike / FunctionRegexpLike implement evaluate_inverted_index: constant pattern -> RegexGramCompiler against the scheme read back from the physical index -> GramQuery -> a candidate bitmap flagged approximate. Approximate results live in their own table in IndexExecContext; SegmentIterator only intersects them into _row_bitmap when the conjunct root is the function itself, never marks the column index-evaluated and keeps the conjunct for re-evaluation, so the result set is always exactly the non-indexed one. NOT / OR-nested predicates skip the evaluation altogether instead of computing a result nothing could read. - gram_boolean_query executes the AND/OR tree on the SNII reader: one batched dictionary resolution per node (each term first checked against the segment's resident bloom filter), memoised entries, one batched posting plan; a gram whose posting was dropped reads as the whole segment. - Cost gate: a node is given up before any posting is read when its estimated candidate count exceeds the budget -- three times the digest's ceiling, or gram_index_max_candidate_ratio_bp basis points of the segment (default 15) on segments of at least gram_index_candidate_ratio_min_rows rows when there is no digest. The AND estimate is a damped product of its grams' selectivities, and a sub-query's estimate feeds its parent. Giving up yields the whole docid range, so it can never prune a row. - Push-down only for SNII readers; CLucene-format readers reject the query type. enable_gram_index_regexp (default true) is the kill switch; RowsGramIndexFiltered, GramIndexCandidateRows and GramIndexGateGaveUp are reported in the scan profile. Tests: gram boolean query unit tests and an end-to-end test over a real SNII segment against brute force, metered-reader assertions on serial read rounds and the gate contracts, like/regexp binding and index evaluation, IndexExecContext isolation, SegmentIterator's approximate-result application and conjunct short-circuit, and the searcher cache. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01METuP3aVRn8dfCU62PivnF --- be/src/exec/operator/olap_scan_operator.cpp | 6 + be/src/exec/operator/olap_scan_operator.h | 4 + be/src/exec/scan/olap_scanner.cpp | 3 + be/src/exprs/function/function.h | 23 + be/src/exprs/function/like.cpp | 100 ++ be/src/exprs/function/like.h | 48 + be/src/exprs/vexpr.cpp | 32 +- be/src/exprs/vexpr_context.h | 46 + .../inverted/inverted_index_query_type.h | 16 +- .../index/inverted/inverted_index_reader.cpp | 11 + .../index/inverted/inverted_index_reader.h | 34 +- .../storage/index/snii/snii_index_reader.cpp | 48 +- be/src/storage/olap_common.h | 10 + be/src/storage/segment/segment_iterator.cpp | 30 + be/src/storage/segment/segment_iterator.h | 4 + .../exprs/function/like_gram_binding_test.cpp | 471 +++++++++ .../exprs/function/like_gram_index_test.cpp | 171 +++ be/test/exprs/index_exec_context_test.cpp | 120 +++ .../query/gram_boolean_query_e2e_test.cpp | 503 +++++++++ .../snii/query/gram_boolean_query_test.cpp | 999 ++++++++++++++++++ .../index/snii/snii_gram_cache_test.cpp | 522 +++++++++ ...t_iterator_conjunct_short_circuit_test.cpp | 148 ++- 22 files changed, 3336 insertions(+), 13 deletions(-) create mode 100644 be/test/exprs/function/like_gram_binding_test.cpp create mode 100644 be/test/exprs/function/like_gram_index_test.cpp create mode 100644 be/test/exprs/index_exec_context_test.cpp create mode 100644 be/test/storage/index/snii/query/gram_boolean_query_e2e_test.cpp create mode 100644 be/test/storage/index/snii/query/gram_boolean_query_test.cpp create mode 100644 be/test/storage/index/snii/snii_gram_cache_test.cpp diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index 621ce2a15764fc..768eedf154c4f7 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -300,6 +300,12 @@ Status OlapScanLocalState::_init_profile() { ADD_COUNTER_WITH_LEVEL(_segment_profile, "InvertedIndexDowngradeCount", TUnit::UNIT, 1); _inverted_index_conjuncts_short_circuited_counter = ADD_COUNTER_WITH_LEVEL( _segment_profile, "InvertedIndexConjunctsShortCircuited", TUnit::UNIT, 1); + _gram_index_filter_counter = + ADD_COUNTER_WITH_LEVEL(_segment_profile, "RowsGramIndexFiltered", TUnit::UNIT, 1); + _gram_index_candidate_counter = + ADD_COUNTER_WITH_LEVEL(_segment_profile, "GramIndexCandidateRows", TUnit::UNIT, 1); + _gram_index_gate_gave_up_counter = + ADD_COUNTER_WITH_LEVEL(_segment_profile, "GramIndexGateGaveUp", TUnit::UNIT, 1); _inverted_index_analyzer_timer = ADD_TIMER_WITH_LEVEL(_segment_profile, "InvertedIndexAnalyzerTime", 1); _inverted_index_lookup_timer = diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index 66d96c024ce5ab..a76783386eb8b8 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -272,6 +272,10 @@ class OlapScanLocalState final : public ScanLocalState { RuntimeProfile::Counter* _inverted_index_searcher_cache_miss_counter = nullptr; RuntimeProfile::Counter* _inverted_index_downgrade_count_counter = nullptr; RuntimeProfile::Counter* _inverted_index_conjuncts_short_circuited_counter = nullptr; + // Approximate (gram) index: rows pruned / candidate rows left for expression re-verification. + RuntimeProfile::Counter* _gram_index_filter_counter = nullptr; + RuntimeProfile::Counter* _gram_index_candidate_counter = nullptr; + RuntimeProfile::Counter* _gram_index_gate_gave_up_counter = nullptr; RuntimeProfile::Counter* _inverted_index_analyzer_timer = nullptr; RuntimeProfile::Counter* _inverted_index_lookup_timer = nullptr; diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 90e9d94a2ae352..9955ae58147d98 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -877,6 +877,9 @@ void OlapScanner::_collect_profile_before_close() { stats.inverted_index_downgrade_count); COUNTER_UPDATE(local_state->_inverted_index_conjuncts_short_circuited_counter, stats.inverted_index_conjuncts_short_circuited); + COUNTER_UPDATE(local_state->_gram_index_filter_counter, stats.rows_gram_index_filtered); + COUNTER_UPDATE(local_state->_gram_index_candidate_counter, stats.gram_index_candidate_rows); + COUNTER_UPDATE(local_state->_gram_index_gate_gave_up_counter, stats.gram_index_gate_gave_up); COUNTER_UPDATE(local_state->_inverted_index_analyzer_timer, stats.inverted_index_analyzer_timer); COUNTER_UPDATE(local_state->_inverted_index_lookup_timer, stats.inverted_index_lookup_timer); diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h index 428059a59d8a57..a19f56b602071e 100644 --- a/be/src/exprs/function/function.h +++ b/be/src/exprs/function/function.h @@ -216,6 +216,21 @@ class IFunctionBase { } } + // Validate operand roles before VExpr separates indexed fields from literals. Functions + // whose index semantics depend on argument order must inspect the original children here. + virtual bool can_evaluate_inverted_index(const VExprSPtrs& /*function_arguments*/) const { + return true; + } + + // True when this function's index push-down can only ever answer with an approximate + // (superset) candidate set instead of the exact row set. Such a result is usable in one + // place only -- narrowing the candidate rows of a conjunct that stays pushed down for + // row-level re-verification -- so VExpr skips the push-down entirely wherever it would be + // computed and then dropped, rather than paying for the index reads first. + virtual bool index_result_is_approximate() const { return false; } + + // VExpr calls this only after can_evaluate_inverted_index accepts the ordered arguments + // and the indexed fields have been bound to storage-compatible types and iterators. virtual Status evaluate_inverted_index( const ColumnsWithTypeAndName& arguments, const std::vector& data_type_with_names, @@ -507,6 +522,14 @@ class DefaultFunction final : public IFunctionBase { return function->close(context, scope); } + bool can_evaluate_inverted_index(const VExprSPtrs& function_arguments) const override { + return function->can_evaluate_inverted_index(function_arguments); + } + + bool index_result_is_approximate() const override { + return function->index_result_is_approximate(); + } + Status evaluate_inverted_index( const ColumnsWithTypeAndName& args, const std::vector& data_type_with_names, diff --git a/be/src/exprs/function/like.cpp b/be/src/exprs/function/like.cpp index c02f9875cc0be5..129ef848bfbf30 100644 --- a/be/src/exprs/function/like.cpp +++ b/be/src/exprs/function/like.cpp @@ -25,14 +25,21 @@ #include #include +#include "common/config.h" +#include "common/exception.h" #include "common/logging.h" #include "core/block/block.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column.h" #include "core/column/column_const.h" #include "core/column/column_vector.h" +#include "core/field.h" #include "core/string_ref.h" #include "exprs/function/simple_function_factory.h" +#include "exprs/vexpr.h" +#include "exprs/vliteral.h" +#include "storage/index/inverted/inverted_index_iterator.h" +#include "storage/index/inverted/inverted_index_reader.h" #include "util/hyperscan_util.h" namespace doris { @@ -1101,6 +1108,99 @@ Status FunctionRegexpLike::open(FunctionContext* context, return Status::OK(); } +bool FunctionLikeBase::can_evaluate_inverted_index(const VExprSPtrs& function_arguments) const { + if (function_arguments.size() != 2 && + !(get_name() == FunctionLike::name && function_arguments.size() == 3)) { + return false; + } + // VExpr subsequently validates that stripping this cast preserves the storage type. + if (!VExpr::expr_without_cast(function_arguments[0])->is_slot_ref() || + !function_arguments[1]->is_literal()) { + return false; + } + if (function_arguments.size() == 3) { + if (!function_arguments[2]->is_literal()) { + return false; + } + const auto* escape = assert_cast(function_arguments[2].get()); + Field escape_field; + escape->get_column_ptr()->get(0, escape_field); + // The gram compiler implements the default LIKE escaping semantics only. + if (escape_field.is_null() || escape_field.get() != "\\") { + return false; + } + } + return true; +} + +// R8 (unity build): file-scope helpers use a namespace private to this file. +namespace like_gram_index_detail { + +// Index acceleration may be skipped, but cancellation and memory failures stop the query. +Status dispatch_query(bool is_like, const std::string& pattern, segment_v2::IndexIterator* iter, + const IndexFieldNameAndTypePair& data_type_with_name, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap* bitmap_result) { + segment_v2::InvertedIndexParam param; + param.column_name = data_type_with_name.first; + param.column_type = data_type_with_name.second; + param.query_value = Field::create_field(pattern); + param.query_type = is_like ? segment_v2::InvertedIndexQueryType::LIKE_GRAM_QUERY + : segment_v2::InvertedIndexQueryType::REGEXP_GRAM_QUERY; + param.num_rows = num_rows; + param.roaring = std::make_shared(); + + Status query_status = iter->read_from_index(¶m); + if (!query_status.ok()) { + if (query_status.is() || + query_status.is() || + query_status.is()) { + return query_status; + } + if (query_status.is() || + query_status.is()) { + return Status::OK(); + } + // Every other error only degrades to "no acceleration". LOG_EVERY_N rather than VLOG, + // because this path already means "the index could not be used" and deserves a trace at + // the default log level, while still not flooding the log once per segment. + LOG_EVERY_N(WARNING, 100) << "gram index push-down skipped, read_from_index returned " + << query_status; + return Status::OK(); + } + + segment_v2::InvertedIndexResultBitmap result(param.roaring, nullptr); + result.set_approximate(true); + *bitmap_result = result; + return Status::OK(); +} + +} // namespace like_gram_index_detail + +Status FunctionLikeBase::evaluate_gram_index( + GramCompileKind kind, const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const { + const bool is_like = (kind == GramCompileKind::LIKE); + if (!config::enable_gram_index_regexp) { + return Status::OK(); + } + // Operand roles were accepted before VExpr separated literals and indexed fields. + DORIS_CHECK_EQ(iterators.size(), 1); + DORIS_CHECK_EQ(data_type_with_names.size(), 1); + DORIS_CHECK(!arguments.empty()); + DORIS_CHECK(is_column_const(*arguments[0].column)); + Field pattern_field; + arguments[0].column->get(0, pattern_field); + if (pattern_field.is_null()) { + return Status::OK(); + } + const auto& pattern = pattern_field.get(); + + return like_gram_index_detail::dispatch_query( + is_like, pattern, iterators[0], data_type_with_names[0], num_rows, &bitmap_result); +} + void register_function_like(SimpleFunctionFactory& factory) { factory.register_function(); } diff --git a/be/src/exprs/function/like.h b/be/src/exprs/function/like.h index 567f59ee1b66d5..8f521282f8bff6 100644 --- a/be/src/exprs/function/like.h +++ b/be/src/exprs/function/like.h @@ -292,6 +292,8 @@ class FunctionLikeBase : public IFunction { Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t /*input_rows_count*/) const override; + bool can_evaluate_inverted_index(const VExprSPtrs& function_arguments) const override; + friend struct VectorAllpassSearchState; friend struct VectorEqualSearchState; friend struct VectorSubStringSearchState; @@ -386,6 +388,24 @@ class FunctionLikeBase : public IFunction { // hyperscan compile expression to database and allocate scratch space static Status hs_prepare(FunctionContext* context, const char* expression, hs_database_t** database, hs_scratch_t** scratch); + + // Send the original LIKE/REGEXP pattern to the selected reader. The reader compiles it + // against its persisted gram scheme and returns an approximate candidate bitmap. + // + // Hard semantic constraint (Rulings R26 / R29): the index may only produce a superset of + // candidates, and any index-side failure or inapplicable case may only cost the speedup -- + // all of them simply return OK() without writing bitmap_result, and a problem on the index + // side must never make a LIKE/REGEXP query fail or change its result. The caller first + // validates the ordered children through can_evaluate_inverted_index: the value is the + // indexed operand, the pattern is a literal, and LIKE ESCAPE is absent or a literal + // backslash. VExpr then binds the single value iterator. This method handles a disabled + // switch, NULL pattern, unsupported index, compiler result of ALL and index errors. + // The only statuses rethrown are CANCELLED / MEM_LIMIT_EXCEEDED / MEM_ALLOC_FAILED. + enum class GramCompileKind { LIKE, REGEXP }; + Status evaluate_gram_index(GramCompileKind kind, const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const; }; class FunctionLike : public FunctionLikeBase { @@ -409,6 +429,20 @@ class FunctionLike : public FunctionLikeBase { friend struct VectorStartsWithSearchState; friend struct VectorEndsWithSearchState; + // The only push-down implemented for LIKE is the gram index, which answers with a superset + // of the matching rows. + bool index_result_is_approximate() const override { return true; } + + Status evaluate_inverted_index( + const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + const InvertedIndexAnalyzerCtx* analyzer_ctx, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const override { + return evaluate_gram_index(GramCompileKind::LIKE, arguments, data_type_with_names, + iterators, num_rows, bitmap_result); + } + private: static Status like_fn(const LikeSearchState* state, const ColumnString& val, const StringRef& pattern, ColumnUInt8::Container& result); @@ -432,6 +466,20 @@ class FunctionRegexpLike : public FunctionLikeBase { String get_name() const override { return name; } Status open(FunctionContext* context, FunctionContext::FunctionStateScope scope) override; + + // The only push-down implemented for REGEXP is the gram index, which answers with a superset + // of the matching rows. + bool index_result_is_approximate() const override { return true; } + + Status evaluate_inverted_index( + const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + const InvertedIndexAnalyzerCtx* analyzer_ctx, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const override { + return evaluate_gram_index(GramCompileKind::REGEXP, arguments, data_type_with_names, + iterators, num_rows, bitmap_result); + } }; } // namespace doris diff --git a/be/src/exprs/vexpr.cpp b/be/src/exprs/vexpr.cpp index e49bb037865582..a036b230295d5d 100644 --- a/be/src/exprs/vexpr.cpp +++ b/be/src/exprs/vexpr.cpp @@ -919,6 +919,22 @@ ColumnPtr VExpr::get_result_from_const(size_t count) const { Status VExpr::_evaluate_inverted_index(VExprContext* context, const FunctionBasePtr& function, uint32_t segment_num_rows) { + if (!function->can_evaluate_inverted_index(children())) { + return Status::OK(); + } + + // A function that can only answer approximately (the gram push-down behind LIKE / REGEXP) + // produces a superset of candidate rows, and the caller names the single expression whose + // superset it will read back. Anywhere else -- an operand of a compound AND/OR/NOT, a + // nested child push-down, a virtual column projection -- the bitmap would be stored and + // never read, so building it would spend term dictionary lookups and posting reads (one + // remote request each on object storage) to filter nothing. Bail out before any of that; + // the row-level path still evaluates the predicate exactly, only the speedup is lost. + if (function->index_result_is_approximate() && + context->approx_index_result_consumer() != this) { + return Status::OK(); + } + // Pre-allocate vectors based on an estimated or known size std::vector iterators; std::vector data_type_with_names; @@ -1036,9 +1052,19 @@ Status VExpr::_evaluate_inverted_index(VExprContext* context, const FunctionBase return res; } if (!result_bitmap.is_empty()) { - index_context->set_index_result_for_expr(this, result_bitmap); - for (int column_id : column_ids) { - index_context->set_true_for_index_status(this, column_id); + if (result_bitmap.approximate()) { + // Approximate (superset) result: it goes only into the approximate map -- not into + // the exact result map (fast_execute would otherwise pass the candidate bitmap off + // as the function result), and it does not set the column's index status to true + // (the column would otherwise be judged not to need its data read, leaving nothing + // for the expression to re-verify against). The expression stays in the push-down + // list and is re-verified by the row-level path. + index_context->set_approx_index_result_for_expr(this, result_bitmap); + } else { + index_context->set_index_result_for_expr(this, result_bitmap); + for (int column_id : column_ids) { + index_context->set_true_for_index_status(this, column_id); + } } } return Status::OK(); diff --git a/be/src/exprs/vexpr_context.h b/be/src/exprs/vexpr_context.h index a1fd41a7518c95..d8093544d351ca 100644 --- a/be/src/exprs/vexpr_context.h +++ b/be/src/exprs/vexpr_context.h @@ -124,6 +124,22 @@ class IndexExecContext { return &iter->second; } + // Dedicated entry point for approximate (superset) index results, kept strictly apart from + // the exact result map above. + void set_approx_index_result_for_expr(const VExpr* expr, + segment_v2::InvertedIndexResultBitmap bitmap) { + _approx_index_result_bitmap[expr] = std::move(bitmap); + } + + const segment_v2::InvertedIndexResultBitmap* get_approx_index_result_for_expr( + const VExpr* expr) const { + auto iter = _approx_index_result_bitmap.find(expr); + if (iter == _approx_index_result_bitmap.end()) { + return nullptr; + } + return &iter->second; + } + void set_index_result_column_for_expr(const VExpr* expr, ColumnPtr column) { _index_result_column[expr] = std::move(column); } @@ -174,6 +190,20 @@ class IndexExecContext { // A map of expressions to their corresponding result columns. std::unordered_map _index_result_column; + // Approximate (superset) index results: rows outside the bitmap certainly do not match, but + // rows inside it may not match either, so it may only be used to prune candidate rows and + // the expression must stay in the push-down list to re-verify them. Three invariants: + // (a) never write into _index_result_bitmap / _index_result_column -- VExpr::fast_execute + // would then pass the candidate bitmap off as the function result, and + // _output_index_result_column would materialize it as a result column; + // (b) never call set_true_for_index_status -- the column would then be judged + // need_read_data=false and there would be no column left to read during re-verification; + // (c) intersecting with _row_bitmap is allowed only when the expression happens to be the + // root of the VExprContext (a top-level AND context); wrapped in NOT/OR, VCompoundPred + // never sees this map, so it simply does not apply. + std::unordered_map + _approx_index_result_bitmap; + // Per-expression analyzer context for inverted index evaluation. std::unordered_map _expr_analyzer_ctx; @@ -246,6 +276,18 @@ class VExprContext { // skip this expr, just do not apply index anymore. [[nodiscard]] Status evaluate_inverted_index(uint32_t segment_num_rows); + // The one expression whose approximate (superset) index result this caller will read back, + // or null when it reads none. SegmentIterator names the root of a pushed-down conjunct, + // because _apply_approx_index_result looks the result up by exactly that pointer; its + // virtual column loop names nothing, because it only materializes exact results. For any + // other expression -- an operand of a compound predicate, a nested child push-down -- the + // result would be stored under its own pointer and never read again, so a function that can + // only answer approximately skips the evaluation instead of paying for the index reads. + void set_approx_index_result_consumer(const VExpr* expr) { + _approx_index_result_consumer = expr; + } + const VExpr* approx_index_result_consumer() const { return _approx_index_result_consumer; } + [[nodiscard]] static ZoneMapFilterResult evaluate_zonemap_filter( const VExprContextSPtrs& conjuncts, const ZoneMapEvalContext& ctx); [[nodiscard]] static ZoneMapFilterResult evaluate_dictionary_filter( @@ -351,6 +393,10 @@ class VExprContext { bool _prepared = false; bool _opened = false; + /// See set_approx_index_result_consumer. Null by default, so a caller that does not read the + /// approximate map never pays for the index reads behind one. + const VExpr* _approx_index_result_consumer = nullptr; + /// FunctionContexts for each registered expression. The FunctionContexts are created /// and owned by this VExprContext. std::vector> _fn_contexts; diff --git a/be/src/storage/index/inverted/inverted_index_query_type.h b/be/src/storage/index/inverted/inverted_index_query_type.h index 00330ebab25a82..87ed0fc86e4b76 100644 --- a/be/src/storage/index/inverted/inverted_index_query_type.h +++ b/be/src/storage/index/inverted/inverted_index_query_type.h @@ -83,6 +83,8 @@ enum class InvertedIndexQueryType { RANGE_QUERY = 13, LIST_QUERY = 14, SEARCH_DSL_QUERY = 15, + LIKE_GRAM_QUERY = 16, + REGEXP_GRAM_QUERY = 17, }; inline bool is_equal_query(InvertedIndexQueryType query_type) { @@ -96,13 +98,19 @@ inline bool is_range_query(InvertedIndexQueryType query_type) { query_type == InvertedIndexQueryType::LESS_EQUAL_QUERY); } +inline bool is_gram_query(InvertedIndexQueryType query_type) { + return query_type == InvertedIndexQueryType::LIKE_GRAM_QUERY || + query_type == InvertedIndexQueryType::REGEXP_GRAM_QUERY; +} + inline bool is_match_query(InvertedIndexQueryType query_type) { return (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY || query_type == InvertedIndexQueryType::MATCH_ALL_QUERY || query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY || query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY || query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY || - query_type == InvertedIndexQueryType::MATCH_PHRASE_EDGE_QUERY); + query_type == InvertedIndexQueryType::MATCH_PHRASE_EDGE_QUERY || + is_gram_query(query_type)); } // Query types that read term positions and so need an index built with support_phrase. @@ -165,6 +173,12 @@ inline std::string query_type_to_string(InvertedIndexQueryType query_type) { case InvertedIndexQueryType::SEARCH_DSL_QUERY: { return "SEARCH_DSL"; } + case InvertedIndexQueryType::LIKE_GRAM_QUERY: { + return "LIKE_GRAM"; + } + case InvertedIndexQueryType::REGEXP_GRAM_QUERY: { + return "REGEXP_GRAM"; + } default: return ""; } diff --git a/be/src/storage/index/inverted/inverted_index_reader.cpp b/be/src/storage/index/inverted/inverted_index_reader.cpp index a8d391ac40f21c..84e2f110b661d8 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.cpp +++ b/be/src/storage/index/inverted/inverted_index_reader.cpp @@ -391,6 +391,12 @@ Status FullTextIndexReader::query(const IndexQueryContextPtr& context, InvertedIndexQueryType query_type, std::shared_ptr& bit_map, const InvertedIndexAnalyzerCtx* analyzer_ctx) { + // CLucene indexes do not persist the gram tokenizer contract needed to compile a pattern. + if (is_gram_query(query_type)) { + return Status::Error( + "{} requires SNII storage format for column {}", query_type_to_string(query_type), + column_name); + } SCOPED_RAW_TIMER(&context->stats->inverted_index_query_timer); std::string search_str = query_value.get(); @@ -520,6 +526,11 @@ Status StringTypeInvertedIndexReader::query(const IndexQueryContextPtr& context, InvertedIndexQueryType query_type, std::shared_ptr& bit_map, const InvertedIndexAnalyzerCtx* /*analyzer_ctx*/) { + if (is_gram_query(query_type)) { + return Status::Error( + "{} requires SNII storage format for column {}", query_type_to_string(query_type), + column_name); + } SCOPED_RAW_TIMER(&context->stats->inverted_index_query_timer); std::string search_str = query_value.get(); diff --git a/be/src/storage/index/inverted/inverted_index_reader.h b/be/src/storage/index/inverted/inverted_index_reader.h index 6960e135e572b5..ab21a2f93e7e31 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.h +++ b/be/src/storage/index/inverted/inverted_index_reader.h @@ -81,6 +81,16 @@ class InvertedIndexResultBitmap { private: std::shared_ptr _data_bitmap = nullptr; std::shared_ptr _null_bitmap = nullptr; + // true means _data_bitmap is only a superset of candidates (from a gram index push-down, for + // instance), so the caller (SegmentIterator) must keep the original expression for row-level + // re-verification and may not consume the expression as it would for an exact index result. + // + // Every operation that produces a new value from an existing one has to carry the flag: the + // four special member functions copy it, and &=, |= and -= OR it in, because a combination + // involving one approximate operand is itself only a superset and must not be mistaken for an + // exact result. op_not is the exception -- see the comment there; negation cannot preserve + // the property at all, so it asserts instead of propagating. + bool _approximate = false; public: // Default constructor @@ -99,12 +109,14 @@ class InvertedIndexResultBitmap { : nullptr), _null_bitmap(other._null_bitmap ? std::make_shared(*other._null_bitmap) - : nullptr) {} + : nullptr), + _approximate(other._approximate) {} // Move constructor InvertedIndexResultBitmap(InvertedIndexResultBitmap&& other) noexcept : _data_bitmap(std::move(other._data_bitmap)), - _null_bitmap(std::move(other._null_bitmap)) {} + _null_bitmap(std::move(other._null_bitmap)), + _approximate(other._approximate) {} // Copy assignment operator InvertedIndexResultBitmap& operator=(const InvertedIndexResultBitmap& other) { @@ -115,6 +127,7 @@ class InvertedIndexResultBitmap { _null_bitmap = other._null_bitmap ? std::make_shared(*other._null_bitmap) : nullptr; + _approximate = other._approximate; } return *this; } @@ -124,6 +137,7 @@ class InvertedIndexResultBitmap { if (this != &other) { // Prevent self-assignment _data_bitmap = std::move(other._data_bitmap); _null_bitmap = std::move(other._null_bitmap); + _approximate = other._approximate; } return *this; } @@ -141,6 +155,7 @@ class InvertedIndexResultBitmap { } *_null_bitmap = std::move(new_null_bitmap); } + _approximate = _approximate || other._approximate; return *this; } @@ -163,11 +178,20 @@ class InvertedIndexResultBitmap { } *_null_bitmap = std::move(new_null_bitmap); } + _approximate = _approximate || other._approximate; return *this; } // NOT operation + // + // Negation is the one combination an approximate result cannot survive: complementing a + // superset of the matching rows yields a subset of the non-matching ones, so rows that do + // match would be dropped, and the approximate flag cannot repair that -- re-verifying the + // survivors never brings back a row that was already excluded. A caller holding an + // approximate result must keep the expression and let the scalar path evaluate the negation + // instead of asking for it here. const InvertedIndexResultBitmap& op_not(const roaring::Roaring* universe) const { + DCHECK(!_approximate) << "op_not on an approximate result would drop matching rows"; if (_data_bitmap) { if (_null_bitmap) { *_data_bitmap = *universe - *_data_bitmap - *_null_bitmap; @@ -190,6 +214,7 @@ class InvertedIndexResultBitmap { *_null_bitmap -= *other._null_bitmap; } } + _approximate = _approximate || other._approximate; return *this; } @@ -206,6 +231,11 @@ class InvertedIndexResultBitmap { // Check if both bitmaps are empty bool is_empty() const { return (_data_bitmap == nullptr && _null_bitmap == nullptr); } + // true = superset of candidates: rows outside the bitmap certainly do not match, but rows + // inside it may not all match, so the expression must re-verify them. + void set_approximate(bool v) { _approximate = v; } + bool approximate() const { return _approximate; } + private: static const roaring::Roaring& _empty_bitmap() { static const roaring::Roaring empty; diff --git a/be/src/storage/index/snii/snii_index_reader.cpp b/be/src/storage/index/snii/snii_index_reader.cpp index ff5c2ffb5cd023..94f77dc1c5086d 100644 --- a/be/src/storage/index/snii/snii_index_reader.cpp +++ b/be/src/storage/index/snii/snii_index_reader.cpp @@ -32,6 +32,7 @@ #include #include +#include "common/cast_set.h" #include "common/config.h" #include "runtime/exec_env.h" #include "runtime/query_context.h" @@ -41,12 +42,15 @@ #include "storage/index/index_reader_helper.h" #include "storage/index/inverted/analyzer/analyzer.h" #include "storage/index/inverted/common/single_flight.h" +#include "storage/index/inverted/gram/gram_query.h" +#include "storage/index/inverted/gram/regex_gram_compiler.h" #include "storage/index/inverted/inverted_index_cache.h" #include "storage/index/inverted/inverted_index_iterator.h" #include "storage/index/snii/format/null_bitmap.h" #include "storage/index/snii/query/boolean_query.h" #include "storage/index/snii/query/count_query.h" #include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/gram_boolean_query.h" #include "storage/index/snii/query/internal/plain_term_routing.h" #include "storage/index/snii/query/phrase_query.h" #include "storage/index/snii/query/prefix_query.h" @@ -130,6 +134,8 @@ struct SniiQueryExecutionResult { std::shared_ptr bitmap; std::vector<::doris::snii::query::PhraseMatch> phrase_matches; bool candidate_rows_consumed = false; + // Gram query nodes the cost gate widened rather than read, for the segment profile. + uint32_t gram_gate_gave_up = 0; }; std::vector to_terms(const InvertedIndexQueryInfo& query_info) { @@ -353,8 +359,8 @@ Status execute_snii_query(const ::doris::snii::reader::LogicalIndexReader& logic const InvertedIndexQueryInfo& query_info, std::string_view search_str, const std::vector& terms, int32_t max_expansions, bool collect_phrase_frequency, SniiQueryExecutionResult* result, - ::doris::snii::query::QueryProfile* profile, - const roaring::Roaring* candidates) { + ::doris::snii::query::QueryProfile* profile, + const roaring::Roaring* candidates, uint64_t rows_of_segment) { result->bitmap = std::make_shared(); result->phrase_matches.clear(); DORIS_CHECK(!collect_phrase_frequency || uses_phrase_frequency_scoring(query_type, query_info)); @@ -427,6 +433,33 @@ Status execute_snii_query(const ::doris::snii::reader::LogicalIndexReader& logic max_expansions); emitted_to_sink = true; break; + case InvertedIndexQueryType::LIKE_GRAM_QUERY: + case InvertedIndexQueryType::REGEXP_GRAM_QUERY: { + // Compile against the same physical dictionary that supplies the postings. Current + // policies can differ from the scheme that was used when this segment was written. + const auto& scheme = logical_reader.gram_scheme(); + if (!scheme.has_value()) { + return Status::Error( + "SNII index is not a gram-family index"); + } + gram::RegexGramCompiler compiler(*scheme); + gram::GramQuery gram_query; + RETURN_IF_ERROR(query_type == InvertedIndexQueryType::LIKE_GRAM_QUERY + ? compiler.compile_like(search_str, &gram_query) + : compiler.compile_regexp(search_str, &gram_query)); + if (gram_query.is_all()) { + return Status::Error( + "Pattern has no prunable grams"); + } + ::doris::snii::query::LogicalIndexPostingSource gram_posting_source(logical_reader); + ::doris::snii::query::GramGateStats gate_stats; + status = ::doris::snii::query::gram_boolean_query(gram_posting_source, gram_query, + cast_set(rows_of_segment), + result->bitmap.get(), &gate_stats); + result->gram_gate_gave_up = gate_stats.nodes_given_up; + emitted_to_sink = true; + break; + } case InvertedIndexQueryType::WILDCARD_QUERY: status = ::doris::snii::query::wildcard_query(logical_reader, search_str, &sink, max_expansions); @@ -482,7 +515,7 @@ Status SniiIndexReader::_parse_query_terms(const IndexQueryContextPtr& context, InvertedIndexQueryInfo* query_info) { DCHECK(query_info != nullptr); if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY || - query_type == InvertedIndexQueryType::WILDCARD_QUERY) { + query_type == InvertedIndexQueryType::WILDCARD_QUERY || is_gram_query(query_type)) { query_info->term_infos.emplace_back(search_str, 0); return Status::OK(); } @@ -848,12 +881,17 @@ Status SniiIndexReader::_compute_query_bitmap( const Status execution_status = execute_snii_query(*logical_reader, query_type, query_info, search_str, *terms, max_expansions, phrase_matches != nullptr, &query_result, - execution_profile.profile(), request.candidates); + execution_profile.profile(), request.candidates, _rows_of_segment); RETURN_IF_ERROR(execution_status); } else { RETURN_IF_ERROR(execute_snii_query(*logical_reader, query_type, query_info, search_str, *terms, max_expansions, phrase_matches != nullptr, - &query_result, nullptr, request.candidates)); + &query_result, nullptr, request.candidates, + _rows_of_segment)); + } + if (query_result.gram_gate_gave_up != 0 && context->stats != nullptr) { + context->stats->gram_index_gate_gave_up += + static_cast(query_result.gram_gate_gave_up); } *out = std::move(query_result.bitmap); context->candidate_rows_consumed = query_result.candidate_rows_consumed; diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 0b05f8fdeb4db8..30899ff271ba4a 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -328,6 +328,16 @@ struct OlapReaderStatistics { // Pushed-down conjuncts skipped (never index-evaluated) because the row // bitmap was already empty when their turn came. int64_t inverted_index_conjuncts_short_circuited = 0; + // Rows pruned by the approximate (gram) index: rows removed when the candidate bitmap is + // intersected with _row_bitmap. + int64_t rows_gram_index_filtered = 0; + // Candidate rows produced by the approximate (gram) index: rows that still need expression + // re-verification after the intersection. + int64_t gram_index_candidate_rows = 0; + // Query nodes the gram index's cost gate widened to the whole segment rather than read. A + // non-zero count is why the two numbers above can show an index that eliminated nothing: it + // was not consulted for those nodes, by a cost decision rather than because no gram matched. + int64_t gram_index_gate_gave_up = 0; int64_t inverted_index_analyzer_timer = 0; int64_t inverted_index_lookup_timer = 0; // See snii_query_stats.h: one field here instead of one per SNII counter. diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 11cda42fece958..fd23c75554e170 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -1397,6 +1397,11 @@ Status SegmentIterator::_apply_index_expr() { break; } ++considered_conjuncts; + // _apply_approx_index_result below reads an approximate (superset) result back by this + // context's root pointer, so name that root as the consumer: a push-down that can only + // answer approximately then knows its work will be used. The virtual column loop + // further down consumes none, and deliberately names none. + expr_ctx->set_approx_index_result_consumer(expr_ctx->root().get()); if (Status st = evaluate_with_candidate_policy(expr_ctx); !st.ok()) { if (_downgrade_without_index(st) || st.code() == ErrorCode::NOT_IMPLEMENTED_ERROR) { continue; @@ -1415,6 +1420,10 @@ Status SegmentIterator::_apply_index_expr() { _row_bitmap &= *result->get_data_bitmap(); consumed_by_index.push_back(expr_ctx.get()); } + } else { + // Approximate results come here: prune the candidates only, never consume the + // expression. + _apply_approx_index_result(expr_ctx.get()); } } @@ -1500,6 +1509,27 @@ Status SegmentIterator::_apply_index_expr() { return Status::OK(); } +void SegmentIterator::_apply_approx_index_result(VExprContext* expr_ctx) { + // An approximate result is a superset of candidates: rows outside the bitmap certainly do + // not match, while rows inside it may not match. So this only narrows _row_bitmap down to + // the candidate set and never records the expression in consumed_by_index -- it must stay in + // _common_expr_ctxs_push_down so that _execute_common_expr re-verifies each candidate row. + const auto* approx = + expr_ctx->get_index_context()->get_approx_index_result_for_expr(expr_ctx->root().get()); + if (approx == nullptr || approx->get_data_bitmap() == nullptr) { + // InvertedIndexResultBitmap allows a null data bitmap (the default-constructed "no + // result" shape). In theory the approximate map only ever holds results with + // is_empty() == false, but dereferencing a null pointer here would be a segfault, which + // costs far more than one extra null check. + return; + } + const uint64_t before = _row_bitmap.cardinality(); + _row_bitmap &= *approx->get_data_bitmap(); + const uint64_t after = _row_bitmap.cardinality(); + _opts.stats->gram_index_candidate_rows += static_cast(after); + _opts.stats->rows_gram_index_filtered += static_cast(before - after); +} + bool SegmentIterator::_count_on_index_fastpath_safe() const { CountOnIndexFastpathFacts facts; facts.is_count_on_index_agg = _opts.push_down_agg_type_opt == TPushAggOp::COUNT_ON_INDEX; diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index 0b85b2d73dedb1..978f00be6ad126 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -160,6 +160,10 @@ class SegmentIterator : public RowwiseIterator { // below the configured engage ratio; refreshed at conjunct boundaries as // earlier index conjuncts shrink the bitmap. No-op once engaged. void _refresh_candidate_pushdown(); + // An approximate (superset) index result is only used to prune _row_bitmap: intersect it and + // account for the two profile counters, while the expression itself must stay in + // _common_expr_ctxs_push_down to be re-verified by _execute_common_expr. + void _apply_approx_index_result(VExprContext* expr_ctx); // G02: true iff answering the single pushed-down MATCH predicate by its // match COUNT alone is indistinguishable from the row-accurate bitmap for // this COUNT_ON_INDEX scan (no deletes, no other filters, full row bitmap, diff --git a/be/test/exprs/function/like_gram_binding_test.cpp b/be/test/exprs/function/like_gram_binding_test.cpp new file mode 100644 index 00000000000000..9dc4ba4804b6e4 --- /dev/null +++ b/be/test/exprs/function/like_gram_binding_test.cpp @@ -0,0 +1,471 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "core/block/block.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/field.h" +#include "exprs/vcompound_pred.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "gen_cpp/AgentService_types.h" +#include "io/fs/local_file_system.h" +#include "runtime/descriptors.h" +#include "runtime/exec_env.h" +#include "runtime/index_policy/index_policy_mgr.h" +#include "runtime/runtime_state.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/index_query_context.h" +#include "storage/index/inverted/inverted_index_cache.h" +#include "storage/index/inverted/inverted_index_iterator.h" +#include "storage/index/snii/snii_index_reader.h" +#include "storage/index/snii/snii_index_writer.h" +#include "storage/olap_common.h" +#include "storage/segment/column_reader.h" +#include "storage/tablet/tablet_schema.h" +#include "util/slice.h" + +namespace doris { +namespace { + +constexpr const char* kTestDir = "./ut_dir/like_gram_binding_test"; +constexpr const char* kIndexPathPrefix = "./ut_dir/like_gram_binding_test/segment"; +constexpr int64_t kIndexId = 6753801; + +struct PatternEvaluation { + ColumnPtr row_result; + segment_v2::InvertedIndexResultBitmap candidates; + bool has_exact_result = false; +}; + +class LikeGramBindingTest : public testing::Test { +protected: + void SetUp() override { + auto* exec_env = ExecEnv::GetInstance(); + _previous_policy_mgr = exec_env->index_policy_mgr(); + exec_env->_index_policy_mgr = &_policy_mgr; + _previous_query_cache = exec_env->get_inverted_index_query_cache(); + _query_cache.reset( + segment_v2::InvertedIndexQueryCache::create_global_cache(1024 * 1024, 1)); + exec_env->set_inverted_index_query_cache(_query_cache.get()); + + TIndexPolicy tokenizer; + tokenizer.id = 6753802; + tokenizer.name = "like_binding_dense_tokenizer"; + tokenizer.type = TIndexPolicyType::TOKENIZER; + tokenizer.properties = {{"type", "ngram"}, {"mode", "dense"}, {"min_gram", "3"}}; + TIndexPolicy analyzer; + analyzer.id = 6753803; + analyzer.name = "like_binding_dense_analyzer"; + analyzer.type = TIndexPolicyType::ANALYZER; + analyzer.properties = {{"tokenizer", tokenizer.name}}; + _policy_mgr.apply_policy_changes({tokenizer, analyzer}, {}); + + TabletIndexPB index_pb; + index_pb.set_index_type(IndexType::INVERTED); + index_pb.set_index_id(kIndexId); + index_pb.set_index_name("like_binding_index"); + index_pb.add_col_unique_id(0); + index_pb.mutable_properties()->insert({"analyzer", analyzer.name}); + _index_meta.init_from_pb(index_pb); + + ASSERT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + ASSERT_TRUE(io::global_local_filesystem()->create_directory(kTestDir).ok()); + auto status = write_indexes(kIndexPathPrefix, {&_index_meta}); + ASSERT_TRUE(status.ok()) << status; + _file_reader = std::make_shared( + io::global_local_filesystem(), kIndexPathPrefix, + InvertedIndexStorageFormatPB::SNII); + status = _file_reader->init(); + ASSERT_TRUE(status.ok()) << status; + auto reader = segment_v2::SniiIndexReader::create_shared( + &_index_meta, _file_reader, segment_v2::InvertedIndexReaderType::FULLTEXT, + _values.size(), /*column_is_array=*/false); + _iterators.resize(1); + status = reader->new_iterator(_iterators.data()); + ASSERT_TRUE(status.ok()) << status; + + TQueryOptions options; + options.__set_query_type(TQueryType::SELECT); + options.__set_enable_hyperscan_fallback(true); + options.__set_enable_inverted_index_query_cache(false); + options.__set_enable_inverted_index_searcher_cache(false); + _runtime_state.set_query_options(options); + _query_context = std::make_shared(); + _query_context->runtime_state = &_runtime_state; + _query_context->stats = &_stats; + _query_context->io_ctx = &_io_context; + _iterators[0]->set_context(_query_context); + } + + void TearDown() override { + _iterators.clear(); + _file_reader.reset(); + auto* exec_env = ExecEnv::GetInstance(); + exec_env->set_inverted_index_query_cache(_previous_query_cache); + _query_cache.reset(); + exec_env->_index_policy_mgr = _previous_policy_mgr; + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + } + + Status write_indexes(const std::string& path_prefix, + const std::vector& indexes) { + io::FileWriterPtr file_writer; + RETURN_IF_ERROR( + io::global_local_filesystem()->create_file(path_prefix + ".idx", &file_writer)); + segment_v2::IndexFileWriter index_file_writer( + io::global_local_filesystem(), path_prefix, "like_binding_rowset", 0, + InvertedIndexStorageFormatPB::SNII, std::move(file_writer), true, kIndexId); + std::vector slices; + for (const auto& value : _values) { + slices.emplace_back(value); + } + for (const auto* index : indexes) { + segment_v2::SniiIndexColumnWriter writer(&index_file_writer, index, + FieldType::OLAP_FIELD_TYPE_VARCHAR); + RETURN_IF_ERROR(writer.init()); + RETURN_IF_ERROR(writer.add_values("p", slices.data(), slices.size())); + RETURN_IF_ERROR(writer.finish()); + } + RETURN_IF_ERROR(index_file_writer.begin_close()); + return index_file_writer.finish_close(); + } + + Status evaluate_pattern(const std::string& name, bool column_on_left, + const std::string& literal_value, PatternEvaluation* result) { + const DataTypePtr string_type = std::make_shared(); + TFunction function; + TFunctionName function_name; + function_name.__set_function_name(name); + function.__set_name(function_name); + function.__set_binary_type(TFunctionBinaryType::BUILTIN); + function.__set_arg_types({string_type->to_thrift(), string_type->to_thrift()}); + function.__set_ret_type(DataTypeUInt8().to_thrift()); + function.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::FUNCTION_CALL); + node.__set_type(DataTypeUInt8().to_thrift()); + node.__set_fn(function); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto expression = VectorizedFnCall::create_shared(node); + auto slot = VSlotRef::create_shared(0, 0, 0, string_type, "p"); + auto literal = VLiteral::create_shared(string_type, + Field::create_field(literal_value)); + if (column_on_left) { + expression->add_child(slot); + expression->add_child(literal); + } else { + expression->add_child(literal); + expression->add_child(slot); + } + + std::vector storage_types {{"p", string_type}}; + std::unordered_map> index_status; + segment_v2::ColumnIteratorOptions column_options; + auto index_context = std::make_shared( + _iterators, storage_types, index_status, nullptr, nullptr, column_options); + VExprContext context(expression); + context.set_index_context(index_context); + RETURN_IF_ERROR(context.prepare(&_runtime_state, RowDescriptor {})); + RETURN_IF_ERROR(context.open(&_runtime_state)); + // SegmentIterator names the conjunct root as the consumer; without that the gram + // push-down knows its result would be dropped and skips the index read altogether. + context.set_approx_index_result_consumer(expression.get()); + + auto column = ColumnString::create(); + for (const auto& value : _values) { + column->insert_data(value.data(), value.size()); + } + Block block; + block.insert({std::move(column), string_type, "p"}); + // Establish the SQL result before any index result can affect expression execution. + RETURN_IF_ERROR(context.execute(&block, result->row_result)); + RETURN_IF_ERROR(context.evaluate_inverted_index(_values.size())); + result->has_exact_result = index_context->has_index_result_for_expr(expression.get()); + if (const auto* candidates = + index_context->get_approx_index_result_for_expr(expression.get())) { + result->candidates = *candidates; + } + return Status::OK(); + } + + void check_selective_result(const PatternEvaluation& result) { + ASSERT_FALSE(result.row_result->get_bool(0)); + ASSERT_TRUE(result.row_result->get_bool(1)); + ASSERT_FALSE(result.row_result->get_bool(2)); + EXPECT_FALSE(result.has_exact_result); + ASSERT_FALSE(result.candidates.is_empty()); + EXPECT_TRUE(result.candidates.approximate()); + EXPECT_TRUE(result.candidates.get_data_bitmap()->contains(1)) + << "The gram index must preserve the TRUE row"; + EXPECT_FALSE(result.candidates.get_data_bitmap()->contains(0)); + EXPECT_FALSE(result.candidates.get_data_bitmap()->contains(2)); + } + + // Where the gram push-down is started from, and whether anybody will read its result back. + enum class PushDownShape { + kConsumedRoot, // the LIKE call is the conjunct root, exactly what SegmentIterator does + kUnconsumedRoot, // same tree, but the caller reads no approximate result (a virtual + // column projection, which only ever materializes exact results) + kUnderNot, // NOT(LIKE ...) + kUnderOr, // LIKE ... OR LIKE ... + }; + + struct PushDownProbe { + int64_t index_reader_opens = 0; // readers opened to answer an index query + bool has_candidates = false; // an approximate result was stored for the LIKE call + }; + + // Opening a reader is the first thing any index query does, so this counts index reads: a + // push-down that was skipped before it started leaves the counters where they were. + int64_t index_reader_opens() const { + return _stats.inverted_index_searcher_cache_hit + _stats.inverted_index_searcher_cache_miss; + } + + Status probe_push_down(PushDownShape shape, PushDownProbe* probe) { + const DataTypePtr string_type = std::make_shared(); + auto make_like = [&]() { + TFunction function; + TFunctionName function_name; + function_name.__set_function_name("like"); + function.__set_name(function_name); + function.__set_binary_type(TFunctionBinaryType::BUILTIN); + function.__set_arg_types({string_type->to_thrift(), string_type->to_thrift()}); + function.__set_ret_type(DataTypeUInt8().to_thrift()); + function.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::FUNCTION_CALL); + node.__set_type(DataTypeUInt8().to_thrift()); + node.__set_fn(function); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto call = VectorizedFnCall::create_shared(node); + call->add_child(VSlotRef::create_shared(0, 0, 0, string_type, "p")); + call->add_child(VLiteral::create_shared( + string_type, Field::create_field(std::string("%abcdef%")))); + return call; + }; + + auto like = make_like(); + VExprSPtr root = like; + if (shape == PushDownShape::kUnderNot || shape == PushDownShape::kUnderOr) { + const bool is_not = shape == PushDownShape::kUnderNot; + TExprNode compound_node; + compound_node.__set_node_type(TExprNodeType::COMPOUND_PRED); + compound_node.__set_type(DataTypeUInt8().to_thrift()); + compound_node.__set_opcode(is_not ? TExprOpcode::COMPOUND_NOT + : TExprOpcode::COMPOUND_OR); + compound_node.__set_num_children(is_not ? 1 : 2); + compound_node.__set_is_nullable(false); + auto compound = VCompoundPred::create_shared(compound_node); + compound->add_child(like); + if (!is_not) { + compound->add_child(make_like()); + } + root = compound; + } + + std::vector storage_types {{"p", string_type}}; + std::unordered_map> index_status; + segment_v2::ColumnIteratorOptions column_options; + auto index_context = std::make_shared( + _iterators, storage_types, index_status, nullptr, nullptr, column_options); + VExprContext context(root); + context.set_index_context(index_context); + RETURN_IF_ERROR(context.prepare(&_runtime_state, RowDescriptor {})); + RETURN_IF_ERROR(context.open(&_runtime_state)); + if (shape != PushDownShape::kUnconsumedRoot) { + // SegmentIterator names the conjunct root for every pushed-down conjunct, compound + // ones included; only the virtual column loop names nothing. + context.set_approx_index_result_consumer(root.get()); + } + + const int64_t before = index_reader_opens(); + RETURN_IF_ERROR(context.evaluate_inverted_index(_values.size())); + probe->index_reader_opens = index_reader_opens() - before; + probe->has_candidates = + index_context->get_approx_index_result_for_expr(like.get()) != nullptr; + return Status::OK(); + } + + const std::vector _values {"%", "abcdef", "unrelated"}; + IndexPolicyMgr _policy_mgr; + IndexPolicyMgr* _previous_policy_mgr = nullptr; + segment_v2::InvertedIndexQueryCache* _previous_query_cache = nullptr; + std::unique_ptr _query_cache; + TabletIndex _index_meta; + std::shared_ptr _file_reader; + std::vector> _iterators; + RuntimeState _runtime_state; + OlapReaderStatistics _stats; + io::IOContext _io_context; + segment_v2::IndexQueryContextPtr _query_context; +}; + +TEST_F(LikeGramBindingTest, LikePreservesOperandRolesAndPrunesIndexedValues) { + PatternEvaluation reverse; + auto status = evaluate_pattern("like", /*column_on_left=*/false, "abcdef", &reverse); + ASSERT_TRUE(status.ok()) << status; + ASSERT_TRUE(reverse.row_result->get_bool(0)); // 'abcdef' LIKE '%' + ASSERT_TRUE(reverse.row_result->get_bool(1)); + ASSERT_FALSE(reverse.row_result->get_bool(2)); + EXPECT_FALSE(reverse.has_exact_result); + EXPECT_TRUE(reverse.candidates.is_empty() || reverse.candidates.get_data_bitmap()->contains(0)) + << "The index on the pattern operand must not discard a TRUE LIKE row"; + + PatternEvaluation forward; + status = evaluate_pattern("like", /*column_on_left=*/true, "%abcdef%", &forward); + ASSERT_TRUE(status.ok()) << status; + check_selective_result(forward); +} + +TEST_F(LikeGramBindingTest, LikeUsesTheWrittenSchemeAfterSameNamePolicyReplacement) { + // SetUp has written and opened a dense3 index. A table can retain that index in the + // recycle bin while its policy names are dropped and recreated with different settings. + TIndexPolicy tokenizer; + tokenizer.id = 6753812; + tokenizer.name = "like_binding_dense_tokenizer"; + tokenizer.type = TIndexPolicyType::TOKENIZER; + tokenizer.properties = {{"type", "ngram"}, {"mode", "dense"}, {"min_gram", "4"}}; + TIndexPolicy analyzer; + analyzer.id = 6753813; + analyzer.name = "like_binding_dense_analyzer"; + analyzer.type = TIndexPolicyType::ANALYZER; + analyzer.properties = {{"tokenizer", tokenizer.name}}; + _policy_mgr.apply_policy_changes({tokenizer, analyzer}, {6753803, 6753802}); + + PatternEvaluation result; + const auto status = evaluate_pattern("like", /*column_on_left=*/true, "%abcdef%", &result); + ASSERT_TRUE(status.ok()) << status; + check_selective_result(result); +} + +TEST_F(LikeGramBindingTest, RegexpPreservesOperandRolesAndPrunesIndexedValues) { + PatternEvaluation reverse; + auto status = evaluate_pattern("regexp", /*column_on_left=*/false, "abcdef%", &reverse); + ASSERT_TRUE(status.ok()) << status; + ASSERT_TRUE(reverse.row_result->get_bool(0)); // 'abcdef%' REGEXP '%' + ASSERT_TRUE(reverse.row_result->get_bool(1)); + ASSERT_FALSE(reverse.row_result->get_bool(2)); + EXPECT_FALSE(reverse.has_exact_result); + EXPECT_TRUE(reverse.candidates.is_empty() || reverse.candidates.get_data_bitmap()->contains(0)) + << "The index on the pattern operand must not discard a TRUE REGEXP row"; + EXPECT_TRUE(reverse.candidates.is_empty() || reverse.candidates.get_data_bitmap()->contains(1)); + + PatternEvaluation forward; + status = evaluate_pattern("regexp", /*column_on_left=*/true, "abcdef", &forward); + ASSERT_TRUE(status.ok()) << status; + check_selective_result(forward); +} + +TEST_F(LikeGramBindingTest, PatternsUseTheSelectedReaderSchemeInASharedContainer) { + TIndexPolicy tokenizer; + tokenizer.id = 6753822; + tokenizer.name = "like_binding_dense4_tokenizer"; + tokenizer.type = TIndexPolicyType::TOKENIZER; + tokenizer.properties = {{"type", "ngram"}, {"mode", "dense"}, {"min_gram", "4"}}; + TIndexPolicy analyzer; + analyzer.id = 6753823; + analyzer.name = "like_binding_dense4_analyzer"; + analyzer.type = TIndexPolicyType::ANALYZER; + analyzer.properties = {{"tokenizer", tokenizer.name}}; + _policy_mgr.apply_policy_changes({tokenizer, analyzer}, {}); + + TabletIndexPB index_pb; + _index_meta.to_schema_pb(&index_pb); + index_pb.set_index_id(100000); + index_pb.set_index_name("like_binding_dense3_index"); + TabletIndex dense3_index; + dense3_index.init_from_pb(index_pb); + index_pb.set_index_id(2000); + index_pb.set_index_name("like_binding_dense4_index"); + (*index_pb.mutable_properties())["analyzer"] = analyzer.name; + TabletIndex dense4_index; + dense4_index.init_from_pb(index_pb); + + const std::string path_prefix = std::string(kTestDir) + "/multiple_indexes"; + auto status = write_indexes(path_prefix, {&dense3_index, &dense4_index}); + ASSERT_TRUE(status.ok()) << status; + auto file_reader = std::make_shared( + io::global_local_filesystem(), path_prefix, InvertedIndexStorageFormatPB::SNII); + status = file_reader->init(); + ASSERT_TRUE(status.ok()) << status; + auto dense3_reader = segment_v2::SniiIndexReader::create_shared( + &dense3_index, file_reader, segment_v2::InvertedIndexReaderType::FULLTEXT, + _values.size(), /*column_is_array=*/false); + auto dense4_reader = segment_v2::SniiIndexReader::create_shared( + &dense4_index, file_reader, segment_v2::InvertedIndexReaderType::FULLTEXT, + _values.size(), /*column_is_array=*/false); + _iterators[0].reset(); + // Registration order differs from selection order: the smaller index id wins. + status = dense3_reader->new_iterator(_iterators.data()); + ASSERT_TRUE(status.ok()) << status; + status = dense4_reader->new_iterator(_iterators.data()); + ASSERT_TRUE(status.ok()) << status; + _iterators[0]->set_context(_query_context); + + for (const auto& [name, pattern] : std::vector> { + {"like", "%abcdef%"}, {"regexp", "abcdef"}}) { + SCOPED_TRACE(name); + PatternEvaluation result; + status = evaluate_pattern(name, /*column_on_left=*/true, pattern, &result); + ASSERT_TRUE(status.ok()) << status; + check_selective_result(result); + } +} + +// A superset of candidate rows is usable in one place only: narrowing the candidates of a +// conjunct that stays pushed down for row-level re-verification. Wherever it is not read back -- +// under NOT / OR, or as a virtual column projection -- the push-down must not run at all. Doing +// the term dictionary lookups and posting reads and then discarding the bitmap is a straight +// performance regression against not having the index in the first place. +TEST_F(LikeGramBindingTest, GramPushDownDoesNoIndexIoWhenItsResultWouldBeDiscarded) { + // Baseline: the shape SegmentIterator actually pushes down does read the index and does + // produce candidates, so the assertions below are about a skipped read and not about an + // index that never works. + PushDownProbe consumed; + ASSERT_TRUE(probe_push_down(PushDownShape::kConsumedRoot, &consumed).ok()); + EXPECT_GT(consumed.index_reader_opens, 0); + EXPECT_TRUE(consumed.has_candidates); + + for (const auto& [name, shape] : std::vector> { + {"virtual column projection", PushDownShape::kUnconsumedRoot}, + {"NOT(LIKE)", PushDownShape::kUnderNot}, + {"LIKE OR LIKE", PushDownShape::kUnderOr}}) { + SCOPED_TRACE(name); + PushDownProbe probe; + ASSERT_TRUE(probe_push_down(shape, &probe).ok()); + EXPECT_EQ(probe.index_reader_opens, 0) << "the index was read for a result nobody can use"; + EXPECT_FALSE(probe.has_candidates); + } +} + +} // namespace +} // namespace doris diff --git a/be/test/exprs/function/like_gram_index_test.cpp b/be/test/exprs/function/like_gram_index_test.cpp new file mode 100644 index 00000000000000..95d9e474ee402c --- /dev/null +++ b/be/test/exprs/function/like_gram_index_test.cpp @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Function boundary tests only. Reader identity and persisted gram schemes are exercised through +// real writer/reader/iterator/expression chains in like_gram_binding_test.cpp. + +#include + +#include +#include +#include +#include + +#include "common/config.h" +#include "common/status.h" +#include "core/column/column_const.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_string.h" +#include "exprs/function/like.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "storage/index/index_iterator.h" +#include "storage/index/inverted/inverted_index_iterator.h" +#include "storage/index/inverted/inverted_index_reader.h" + +namespace doris { +namespace { + +// Supplies a result or Status at the function's iterator boundary. It owns no index metadata, +// reader or policy and makes no claim about gram compilation or index capabilities. +class RecordingGramIndexIterator : public segment_v2::IndexIterator { +public: + segment_v2::IndexReaderPtr get_reader(segment_v2::IndexReaderType) const override { + return nullptr; + } + + Status read_from_index(const segment_v2::IndexParam& param) override { + queried = true; + RETURN_IF_ERROR(query_status); + auto* request = std::get(param); + request->roaring->add(3); + request->roaring->add(5); + return Status::OK(); + } + + Status read_null_bitmap(segment_v2::InvertedIndexQueryCacheHandle*) override { + return Status::OK(); + } + + Result has_null() override { return false; } + + Status query_status = Status::OK(); + bool queried = false; +}; + +ColumnWithTypeAndName const_string_arg(const std::string& value) { + auto col = ColumnString::create(); + col->insert_data(value.data(), value.size()); + return {ColumnConst::create(std::move(col), 1), std::make_shared(), "arg"}; +} + +TEST(LikeGramIndexTest, SuccessfulIndexResultIsApproximate) { + const std::vector functions {FunctionLike::create(), FunctionRegexpLike::create()}; + ColumnsWithTypeAndName args {const_string_arg("hello")}; + std::vector names {{"msg", std::make_shared()}}; + for (const auto& function : functions) { + RecordingGramIndexIterator iterator; + segment_v2::InvertedIndexResultBitmap result; + const auto status = + function->evaluate_inverted_index(args, names, {&iterator}, 100, nullptr, result); + ASSERT_TRUE(status.ok()) << status; + EXPECT_TRUE(result.approximate()); + ASSERT_NE(result.get_data_bitmap(), nullptr); + EXPECT_EQ(result.get_data_bitmap()->cardinality(), 2U); + EXPECT_TRUE(result.get_data_bitmap()->contains(3)); + EXPECT_TRUE(result.get_data_bitmap()->contains(5)); + } +} + +TEST(LikeGramIndexTest, CustomEscapeIsNotPushedDown) { + FunctionLike function; + const DataTypePtr type = std::make_shared(); + auto value = VSlotRef::create_shared(0, 0, 0, type, "value"); + auto pattern = VLiteral::create_shared(type, Field::create_field("%abcd%")); + auto custom_escape = VLiteral::create_shared(type, Field::create_field("#")); + auto default_escape = VLiteral::create_shared(type, Field::create_field("\\")); + auto variable_escape = VSlotRef::create_shared(1, 1, 1, type, "escape"); + + // Eligibility is checked while every operand still has its original position, including + // an ESCAPE column that has no index and would disappear from the flattened arguments. + EXPECT_FALSE(function.can_evaluate_inverted_index({value, pattern, custom_escape})); + EXPECT_FALSE(function.can_evaluate_inverted_index({value, pattern, variable_escape})); + EXPECT_FALSE( + function.can_evaluate_inverted_index({value, pattern, default_escape, custom_escape})); + EXPECT_TRUE(function.can_evaluate_inverted_index({value, pattern, default_escape})); +} + +TEST(LikeGramIndexTest, ConfigDisabledSkipsPushDown) { + RecordingGramIndexIterator iterator; + FunctionRegexpLike function; + std::vector names {{"msg", std::make_shared()}}; + segment_v2::InvertedIndexResultBitmap result; + ColumnsWithTypeAndName args {const_string_arg("hello|world")}; + + const bool saved_enable = config::enable_gram_index_regexp; + config::enable_gram_index_regexp = false; + const Status status = + function.evaluate_inverted_index(args, names, {&iterator}, 100, nullptr, result); + config::enable_gram_index_regexp = saved_enable; + + ASSERT_TRUE(status.ok()) << status; + EXPECT_TRUE(result.is_empty()); + EXPECT_FALSE(iterator.queried); +} + +TEST(LikeGramIndexTest, ArbitraryIndexErrorDegradesToNoResult) { + FunctionRegexpLike function; + std::vector names {{"msg", std::make_shared()}}; + ColumnsWithTypeAndName args {const_string_arg("hello|world")}; + const std::vector degradable { + Status::InternalError("boom"), + Status::Error("not supported"), + Status::Error("no prunable grams"), + Status::Error("corrupt index image"), + Status::Error("s3 read failed"), + }; + for (const auto& injected : degradable) { + RecordingGramIndexIterator iterator; + iterator.query_status = injected; + segment_v2::InvertedIndexResultBitmap result; + const Status status = + function.evaluate_inverted_index(args, names, {&iterator}, 100, nullptr, result); + ASSERT_TRUE(status.ok()) << injected.to_string() << " -> " << status.to_string(); + EXPECT_TRUE(result.is_empty()) << injected.to_string(); + } +} + +TEST(LikeGramIndexTest, CancellationAndMemoryErrorsPropagate) { + FunctionRegexpLike function; + std::vector names {{"msg", std::make_shared()}}; + ColumnsWithTypeAndName args {const_string_arg("hello|world")}; + const std::vector propagated {ErrorCode::CANCELLED, ErrorCode::MEM_LIMIT_EXCEEDED, + ErrorCode::MEM_ALLOC_FAILED}; + for (const int code : propagated) { + RecordingGramIndexIterator iterator; + iterator.query_status = Status(code, "stop the query"); + segment_v2::InvertedIndexResultBitmap result; + const Status status = + function.evaluate_inverted_index(args, names, {&iterator}, 100, nullptr, result); + ASSERT_FALSE(status.ok()) << code; + EXPECT_EQ(status.code(), code); + EXPECT_TRUE(result.is_empty()); + } +} + +} // namespace +} // namespace doris diff --git a/be/test/exprs/index_exec_context_test.cpp b/be/test/exprs/index_exec_context_test.cpp new file mode 100644 index 00000000000000..660530186459f6 --- /dev/null +++ b/be/test/exprs/index_exec_context_test.cpp @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "storage/index/inverted/inverted_index_reader.h" +#include "storage/olap_common.h" + +namespace doris::index_exec_context_test { + +// Only used to provide distinct "expression identity" keys: IndexExecContext treats a VExpr* as a +// map key and never dereferences it, so there is no need to build a real expression tree here. +class StubVExpr final : public VExpr { +public: + const std::string& expr_name() const override { return _name; } + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::NotSupported("StubVExpr is identity-only"); + } + +private: + const std::string _name = "stub_vexpr"; +}; + +// Build a candidate bitmap holding the single row row_id. +segment_v2::InvertedIndexResultBitmap make_bitmap(uint32_t row_id, bool approximate) { + auto data_bitmap = std::make_shared(); + data_bitmap->add(row_id); + segment_v2::InvertedIndexResultBitmap result(std::move(data_bitmap), nullptr); + result.set_approximate(approximate); + return result; +} + +// An approximate result must be fully isolated from the exact tables: it may not enter +// _index_result_bitmap / _index_result_column and may not touch a column's index execution +// status, or fast_execute would take the candidates for the result, or the column would be judged +// not to need its data read. +TEST(IndexExecContextTest, ApproxResultIsIsolatedFromExactTables) { + const std::vector> iterators; + const std::vector storage_name_and_type; + std::unordered_map> index_status; + const segment_v2::ColumnIteratorOptions column_iter_opts; + IndexExecContext ctx(iterators, storage_name_and_type, index_status, nullptr, nullptr, + column_iter_opts); + + StubVExpr expr; + ctx.set_approx_index_result_for_expr(&expr, make_bitmap(7, true)); + + const auto* approx = ctx.get_approx_index_result_for_expr(&expr); + ASSERT_NE(approx, nullptr); + EXPECT_TRUE(approx->approximate()); + EXPECT_TRUE(approx->get_data_bitmap()->contains(7)); + + // All three tables on the exact side must be empty. + EXPECT_FALSE(ctx.has_index_result_for_expr(&expr)); + EXPECT_EQ(ctx.get_index_result_for_expr(&expr), nullptr); + EXPECT_TRUE(ctx.get_index_result_bitmap().empty()); + EXPECT_TRUE(ctx.get_index_result_column().empty()); + EXPECT_TRUE(index_status.empty()); + + // An expression that was never written cannot be found in the approximate table. + StubVExpr other; + EXPECT_EQ(ctx.get_approx_index_result_for_expr(&other), nullptr); +} + +// The other half of the split: an exact result still enters the exact table and sets the column +// status to true; an approximate result flips no column status. +TEST(IndexExecContextTest, ExactResultKeepsExactTableAndColumnStatus) { + const std::vector> iterators; + const std::vector storage_name_and_type; + StubVExpr exact_expr; + StubVExpr approx_expr; + constexpr ColumnId kColumnId = 3; + std::unordered_map> index_status; + index_status[kColumnId][&exact_expr] = false; + index_status[kColumnId][&approx_expr] = false; + const segment_v2::ColumnIteratorOptions column_iter_opts; + IndexExecContext ctx(iterators, storage_name_and_type, index_status, nullptr, nullptr, + column_iter_opts); + + // The exact path: the same shape as in VExpr::evaluate_inverted_index. + ctx.set_index_result_for_expr(&exact_expr, make_bitmap(11, false)); + ctx.set_true_for_index_status(&exact_expr, static_cast(kColumnId)); + EXPECT_TRUE(ctx.has_index_result_for_expr(&exact_expr)); + ASSERT_NE(ctx.get_index_result_for_expr(&exact_expr), nullptr); + EXPECT_FALSE(ctx.get_index_result_for_expr(&exact_expr)->approximate()); + EXPECT_TRUE(index_status[kColumnId][&exact_expr]); + EXPECT_EQ(ctx.get_approx_index_result_for_expr(&exact_expr), nullptr); + + // The approximate path: only the approximate table is written, the column status stays false. + ctx.set_approx_index_result_for_expr(&approx_expr, make_bitmap(12, true)); + EXPECT_FALSE(ctx.has_index_result_for_expr(&approx_expr)); + EXPECT_FALSE(index_status[kColumnId][&approx_expr]); + EXPECT_EQ(ctx.get_index_result_bitmap().size(), 1U); + EXPECT_TRUE(ctx.get_index_result_column().empty()); +} + +} // namespace doris::index_exec_context_test diff --git a/be/test/storage/index/snii/query/gram_boolean_query_e2e_test.cpp b/be/test/storage/index/snii/query/gram_boolean_query_e2e_test.cpp new file mode 100644 index 00000000000000..3f05951a9a550a --- /dev/null +++ b/be/test/storage/index/snii/query/gram_boolean_query_e2e_test.cpp @@ -0,0 +1,503 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// End-to-end test of the gram boolean query: a batch of real text is written through the **write +// side** gram-family analyzer into a real SNII segment (SniiIndexColumnWriter -> IndexFileWriter +// -> .idx file), reopened with LogicalIndexReader, and a set of LIKE / REGEXP patterns is run +// through "RegexGramCompiler compilation -> gram_boolean_query evaluation" and compared against a +// brute-force truth. +// +// This is the only test on the whole path that pins **phase B's docid space and dictionary keys** +// together with **phase C's lookups**: +// * dictionary keys: on the write side GramTokenizer produces gram tokens that become SNII +// dictionary terms; on the query side RegexGramCompiler derives gram literals with the same +// GramScheme and looks them up in that dictionary. Any drift in encoding, case folding or +// hash sampling on either side shows up here as a false negative ("the truth matches but the +// bitmap does not contain the row"). +// * docid space: the segment holds both empty-string rows and NULL rows, and they occupy docids +// as well. If the write side ever loses a row (add_nulls not advancing rid, say), every later +// row's docid shifts and this test fails too. +// +// There is only one direction to assert: the bitmap must be a **superset** of the truth set +// (truth => candidate). A gram index only narrows candidates, and the extra candidate rows are +// re-verified row by row by the expression above it, so "too many" is allowed and "missing" is +// fatal. + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/status.h" +#include "gen_cpp/AgentService_types.h" +#include "io/fs/local_file_system.h" +#include "runtime/exec_env.h" +#include "runtime/index_policy/index_policy_mgr.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/inverted/gram/gram_family.h" +#include "storage/index/inverted/gram/gram_query.h" +#include "storage/index/inverted/gram/gram_scheme.h" +#include "storage/index/inverted/gram/regex_gram_compiler.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/query/gram_boolean_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/snii_index_writer.h" +#include "storage/tablet/tablet_schema.h" +#include "util/slice.h" + +namespace doris::segment_v2 { +namespace { + +constexpr const char* kTestDir = "./ut_dir/gram_boolean_query_e2e_test"; +constexpr int64_t kIndexId = 9201; + +void assert_ok(const Status& status) { + ASSERT_TRUE(status.ok()) << status.to_string(); +} + +// Pins both knobs of gram_boolean_query()'s candidate cost gate for one scope: the ratio and the +// row floor below which the ratio is not applied at all. This corpus is 30 rows, far under the +// production floor, so the gate can only be made to fire by lowering the floor deliberately. +class ScopedCandidateRatio { +public: + ScopedCandidateRatio(int32_t bp, int32_t min_rows) + : _saved_bp(config::gram_index_max_candidate_ratio_bp), + _saved_min_rows(config::gram_index_candidate_ratio_min_rows) { + config::gram_index_max_candidate_ratio_bp = bp; + config::gram_index_candidate_ratio_min_rows = min_rows; + } + ScopedCandidateRatio(const ScopedCandidateRatio&) = delete; + ScopedCandidateRatio& operator=(const ScopedCandidateRatio&) = delete; + ScopedCandidateRatio(ScopedCandidateRatio&&) = delete; + ScopedCandidateRatio& operator=(ScopedCandidateRatio&&) = delete; + ~ScopedCandidateRatio() { + config::gram_index_max_candidate_ratio_bp = _saved_bp; + config::gram_index_candidate_ratio_min_rows = _saved_min_rows; + } + +private: + int32_t _saved_bp; + int32_t _saved_min_rows; +}; + +// Inject two gram-family policy sets (sparse and dense), the same way as ScopedGramPolicies in +// snii_writer_test.cpp: swap ExecEnv::_index_policy_mgr and restore it on destruction. The ids +// and names are private to this file. IndexPolicyMgr's namespace is shared across policy types, +// so a tokenizer and an analyzer cannot have the same name. +class ScopedGramE2EPolicies { +public: + ScopedGramE2EPolicies() { + auto* exec_env = ExecEnv::GetInstance(); + _previous = exec_env->index_policy_mgr(); + exec_env->_index_policy_mgr = &_manager; + + TIndexPolicy sparse_tokenizer; + sparse_tokenizer.id = 9201; + sparse_tokenizer.name = "gram_e2e_sparse_tokenizer"; + sparse_tokenizer.type = TIndexPolicyType::TOKENIZER; + sparse_tokenizer.properties["type"] = "ngram"; + sparse_tokenizer.properties["mode"] = "sparse"; + + TIndexPolicy sparse_analyzer; + sparse_analyzer.id = 9202; + sparse_analyzer.name = sparse_analyzer_name(); + sparse_analyzer.type = TIndexPolicyType::ANALYZER; + sparse_analyzer.properties["tokenizer"] = sparse_tokenizer.name; + + TIndexPolicy dense_tokenizer; + dense_tokenizer.id = 9203; + dense_tokenizer.name = "gram_e2e_dense_tokenizer"; + dense_tokenizer.type = TIndexPolicyType::TOKENIZER; + dense_tokenizer.properties["type"] = "ngram"; + dense_tokenizer.properties["mode"] = "dense"; + dense_tokenizer.properties["min_gram"] = "3"; + + TIndexPolicy dense_analyzer; + dense_analyzer.id = 9204; + dense_analyzer.name = dense_analyzer_name(); + dense_analyzer.type = TIndexPolicyType::ANALYZER; + dense_analyzer.properties["tokenizer"] = dense_tokenizer.name; + + _manager.apply_policy_changes( + {sparse_tokenizer, sparse_analyzer, dense_tokenizer, dense_analyzer}, {}); + } + + ~ScopedGramE2EPolicies() { ExecEnv::GetInstance()->_index_policy_mgr = _previous; } + + static std::string sparse_analyzer_name() { return "gram_e2e_sparse"; } + static std::string dense_analyzer_name() { return "gram_e2e_dense"; } + + IndexPolicyMgr& manager() { return _manager; } + +private: + IndexPolicyMgr _manager; + IndexPolicyMgr* _previous = nullptr; +}; + +std::map gram_index_properties(const std::string& analyzer_name) { + return {{"analyzer", analyzer_name}}; +} + +TabletIndex make_index_meta(const std::map& properties) { + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(kIndexId); + pb.set_index_name("gram_e2e_idx"); + pb.add_col_unique_id(0); + for (const auto& [key, value] : properties) { + pb.mutable_properties()->insert({key, value}); + } + TabletIndex meta; + meta.init_from_pb(pb); + return meta; +} + +// The corpus: about 30 short lines mixing log-like ASCII rows, CJK rows, empty strings and NULLs. +// A nullopt marks a NULL row -- it occupies a docid too, and is the key probe for docid space +// alignment. +std::vector> build_corpus() { + return { + "rpc error: code = Unavailable desc = timeout", + "rpc error: code = DeadlineExceeded", + "GET /images/x.gif 200 12ms", + "POST /api/v1/login 500 user_id=abc", + std::nullopt, + "手机微博登录失败", + "微博 POST 10.68.3.18:8080 error", + "", + "hello world", + "hello doris", + "world without hello", + "abcdefg starts here", + "prefix abcdefg middle", + "xyz at the very start", + "ends with xyz", + "phone: 010-12345678", + "ticket 123-4567 issued", + "no digits at all here", + std::nullopt, + "手机号码 138-0000-0000", + "Convert conversion successful", + "timeout after error error error", + "code = Unavailable", + "code = OK", + "", + "aaa bbb ccc", + "用户 user_id=42 手机 GET", + "err0r: code = Unknown", + "errxr: computed", + "final row without markers", + }; +} + +// Feed the corpus to a real SniiIndexColumnWriter and produce a real .idx file. Consecutive +// non-NULL rows form one add_values batch and consecutive NULL rows one add_nulls batch, matching +// how SegmentWriter calls the writer in production. +void write_segment(const std::string& path, const TabletIndex& index_meta, + const std::vector>& corpus) { + io::FileWriterPtr file_writer; + assert_ok(io::global_local_filesystem()->create_file(path, &file_writer)); + IndexFileWriter index_file_writer(io::global_local_filesystem(), path, "gram_e2e_rowset", + /*seg_id=*/0, InvertedIndexStorageFormatPB::SNII, + std::move(file_writer), /*can_use_ram_dir=*/true, + /*tablet_id=*/9201); + SniiIndexColumnWriter writer(&index_file_writer, &index_meta, + FieldType::OLAP_FIELD_TYPE_VARCHAR); + assert_ok(writer.init()); + + size_t i = 0; + while (i < corpus.size()) { + if (!corpus[i].has_value()) { + size_t run = 0; + while (i + run < corpus.size() && !corpus[i + run].has_value()) { + ++run; + } + assert_ok(writer.add_nulls(static_cast(run))); + i += run; + continue; + } + std::vector slices; + while (i < corpus.size() && corpus[i].has_value()) { + slices.emplace_back(*corpus[i]); + ++i; + } + assert_ok(writer.add_values("c1", slices.data(), slices.size())); + } + + assert_ok(writer.finish()); + assert_ok(index_file_writer.begin_close()); + assert_ok(index_file_writer.finish_close()); +} + +// One atom of a LIKE pattern with escapes already expanded: % , _ , or an ordinary character. +struct LikeAtom { + bool any_seq = false; // % + bool any_one = false; // _ + char ch = 0; +}; + +std::vector parse_like_pattern(const std::string& pattern) { + std::vector atoms; + for (size_t p = 0; p < pattern.size(); ++p) { + const char c = pattern[p]; + if (c == '\\' && p + 1 < pattern.size() && + (pattern[p + 1] == '%' || pattern[p + 1] == '_' || pattern[p + 1] == '\\')) { + atoms.push_back({.any_seq = false, .any_one = false, .ch = pattern[p + 1]}); + ++p; + } else if (c == '%') { + atoms.push_back({.any_seq = true, .any_one = false, .ch = 0}); + } else if (c == '_') { + atoms.push_back({.any_seq = false, .any_one = true, .ch = 0}); + } else { + atoms.push_back({.any_seq = false, .any_one = false, .ch = c}); + } + } + return atoms; +} + +// LIKE truth: whole-string matching supporting % / _ and the \% \_ \\ escapes. A classic O(n*m) +// dynamic program that deliberately reuses none of the split logic under test -- it has to be an +// independent source of truth. +// `_` matches a single byte: the patterns in this file only use `_` between pure ASCII literals, +// and in UTF-8 no continuation byte of a multi-byte character can sit between two ASCII bytes, so +// on this corpus that is equivalent to the "single character" semantics. +bool like_match(const std::string& value, const std::string& pattern) { + const std::vector atoms = parse_like_pattern(pattern); + const size_t n = value.size(); + const size_t k = atoms.size(); + std::vector prev(k + 1, 0); + std::vector cur(k + 1, 0); + prev[0] = 1; + for (size_t j = 1; j <= k; ++j) { + prev[j] = static_cast(atoms[j - 1].any_seq && prev[j - 1] != 0); + } + for (size_t i = 1; i <= n; ++i) { + cur[0] = 0; + for (size_t j = 1; j <= k; ++j) { + const LikeAtom& a = atoms[j - 1]; + if (a.any_seq) { + cur[j] = static_cast(cur[j - 1] != 0 || prev[j] != 0); + } else if (a.any_one) { + cur[j] = prev[j - 1]; + } else { + cur[j] = static_cast(prev[j - 1] != 0 && value[i - 1] == a.ch); + } + } + prev.swap(cur); + } + return prev[k] != 0; +} + +struct PatternCase { + bool is_like = false; + const char* pattern = nullptr; +}; + +const std::vector& pattern_cases() { + static const std::vector cases { + {.is_like = true, .pattern = "%code = U%"}, + {.is_like = true, .pattern = "%手机%"}, + {.is_like = true, .pattern = "abc%"}, + {.is_like = true, .pattern = "%xyz%"}, + {.is_like = true, .pattern = "%err_r%"}, + {.is_like = true, .pattern = "%hello%world%"}, + {.is_like = false, .pattern = "err.r: co"}, + {.is_like = false, .pattern = "(手|微)博"}, + {.is_like = false, .pattern = "[0-9]{3}-[0-9]{4}"}, + {.is_like = false, .pattern = "hello|world"}, + {.is_like = false, .pattern = "Unavailable"}, + {.is_like = false, .pattern = "^rpc error"}, + }; + return cases; +} + +// Brute-force truth: decide the match for each row independently. A NULL row never matches. +roaring::Roaring brute_force_truth(const std::vector>& corpus, + const PatternCase& pattern_case) { + roaring::Roaring truth; + std::unique_ptr rx; + if (!pattern_case.is_like) { + rx = std::make_unique(pattern_case.pattern, RE2::Quiet); + EXPECT_TRUE(rx->ok()) << pattern_case.pattern; + } + for (uint32_t docid = 0; docid < corpus.size(); ++docid) { + if (!corpus[docid].has_value()) { + continue; + } + const std::string& value = *corpus[docid]; + const bool matched = pattern_case.is_like ? like_match(value, pattern_case.pattern) + : RE2::PartialMatch(value, *rx); + if (matched) { + truth.add(docid); + } + } + return truth; +} + +roaring::Roaring full_range(uint32_t num_docs) { + roaring::Roaring full; + full.addRange(0, num_docs); + return full; +} + +// Run "compile -> evaluate -> compare against the truth" for one pattern. Extracted into a +// function to keep the cognitive complexity of run_scheme() down (clang-tidy +// readability-function-cognitive-complexity). +void check_pattern(const std::vector>& corpus, + const doris::snii::reader::LogicalIndexReader& index, + gram::RegexGramCompiler& compiler, const std::string& analyzer_name, + const PatternCase& pattern_case, int* indexable) { + const auto num_docs = static_cast(corpus.size()); + gram::GramQuery query; + const Status compile_status = pattern_case.is_like + ? compiler.compile_like(pattern_case.pattern, &query) + : compiler.compile_regexp(pattern_case.pattern, &query); + ASSERT_TRUE(compile_status.ok()) << pattern_case.pattern; + + // The truth comparison below runs on the PRODUCTION gate defaults. This corpus is far under + // the configured row floor, so the cost gate is structurally unable to fire and the candidate + // set really is the pruned one -- which matters, because a gate that widened everything to + // the whole segment would satisfy the "no false negative" assertion trivially and this is the + // one test that proves the index never loses a matching row. + ASSERT_LT(num_docs, static_cast(config::gram_index_candidate_ratio_min_rows)); + roaring::Roaring bitmap; + { + doris::snii::query::LogicalIndexPostingSource source(index); + ASSERT_TRUE(doris::snii::query::gram_boolean_query(source, query, num_docs, &bitmap).ok()) + << pattern_case.pattern; + } + { + // Same pattern with the gate deliberately forced on (floor dropped under this corpus, 1% + // ratio): every gram is above 1% here, so a node with grams gives up and yields the whole + // segment. Whatever it returns must be a superset of the ungated candidate set -- giving + // up may only add rows. + ScopedCandidateRatio ratio(/*bp=*/100, /*min_rows=*/1); + doris::snii::query::LogicalIndexPostingSource source(index); + roaring::Roaring gated; + ASSERT_TRUE(doris::snii::query::gram_boolean_query(source, query, num_docs, &gated).ok()) + << pattern_case.pattern; + EXPECT_TRUE((bitmap - gated).isEmpty()) + << "the candidate ratio gate dropped rows for " << pattern_case.pattern; + EXPECT_TRUE((gated - full_range(num_docs)).isEmpty()) << pattern_case.pattern; + } + + if (query.is_all()) { + // ALL has to let the entire docid space through, otherwise rows would be pruned with no + // information at all. + EXPECT_TRUE(bitmap == full_range(num_docs)) << pattern_case.pattern; + } else { + ++(*indexable); + } + // The candidate set always stays within [0, num_docs). + EXPECT_TRUE((bitmap - full_range(num_docs)).isEmpty()) << pattern_case.pattern; + + const roaring::Roaring truth = brute_force_truth(corpus, pattern_case); + const roaring::Roaring missed = truth - bitmap; + EXPECT_TRUE(missed.isEmpty()) << "FALSE NEGATIVE analyzer=" << analyzer_name + << " pattern=" << pattern_case.pattern + << " q=" << query.to_debug_string() + << " missed=" << missed.toString(); +} + +// Run the full "write segment -> open segment -> compile -> evaluate -> compare truth" for one +// gram scheme. +void run_scheme(const std::string& analyzer_name, const std::string& segment_name) { + ScopedGramE2EPolicies policies; + const auto properties = gram_index_properties(analyzer_name); + const TabletIndex index_meta = make_index_meta(properties); + const auto corpus = build_corpus(); + const auto num_docs = static_cast(corpus.size()); + + const std::string path = std::string(kTestDir) + "/" + segment_name + ".idx"; + write_segment(path, index_meta, corpus); + if (::testing::Test::HasFatalFailure()) { + return; + } + + // The query side resolves the scheme exactly as production does: index properties plus the + // policy manager, nothing else. + const std::optional scheme = + gram::resolve_gram_scheme(properties, &policies.manager()); + ASSERT_TRUE(scheme.has_value()) << analyzer_name; + + doris::snii::io::LocalFileReader file; + assert_ok(file.open(path)); + doris::snii::reader::SniiSegmentReader segment; + assert_ok(doris::snii::reader::SniiSegmentReader::open(&file, &segment)); + ASSERT_EQ(segment.n_logical_indexes(), 1U); + doris::snii::reader::LogicalIndexReader index; + assert_ok(segment.open_index(static_cast(kIndexId), "", &index)); + // The docid space must cover every row, empty-string and NULL rows included. + ASSERT_EQ(index.stats().doc_count, num_docs); + + gram::RegexGramCompiler compiler(*scheme); + int indexable = 0; + for (const PatternCase& pattern_case : pattern_cases()) { + check_pattern(corpus, index, compiler, analyzer_name, pattern_case, &indexable); + if (::testing::Test::HasFatalFailure()) { + return; + } + } + // Coverage guard: at least a third of the patterns that CAN prune must actually do so, + // otherwise this test degenerates into "everything is ALL and nothing is verified". The + // threshold is a third rather than a half because a SPARSE scheme keeps grams by hash + // sampling, so the share of short literals it can reliably pin down is naturally lower + // than DENSE's; what this guards against is "the compiler or the write side broke + // entirely", not the difference in pruning power between the two schemes. + // + // Non-ASCII patterns are excluded from the denominator because the index declines that + // text outright: they are expected to compile to ALL and are still checked above for + // returning exactly what a brute-force scan returns, which is the property that matters. + int prunable_candidates = 0; + for (const PatternCase& pattern_case : pattern_cases()) { + const std::string_view p = pattern_case.pattern; + if (std::ranges::all_of(p, [](char c) { return static_cast(c) < 0x80; })) { + ++prunable_candidates; + } + } + EXPECT_GE(indexable, prunable_candidates / 3) + << "analyzer=" << analyzer_name << ", ascii patterns=" << prunable_candidates; +} + +class GramBooleanQueryE2E : public testing::Test { +protected: + void SetUp() override { + assert_ok(io::global_local_filesystem()->delete_directory(kTestDir)); + assert_ok(io::global_local_filesystem()->create_directory(kTestDir)); + } + void TearDown() override { + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + } +}; + +TEST_F(GramBooleanQueryE2E, SparseSchemeMatchesBruteForce) { + run_scheme(ScopedGramE2EPolicies::sparse_analyzer_name(), "sparse"); +} + +TEST_F(GramBooleanQueryE2E, DenseSchemeMatchesBruteForce) { + run_scheme(ScopedGramE2EPolicies::dense_analyzer_name(), "dense"); +} + +} // namespace +} // namespace doris::segment_v2 diff --git a/be/test/storage/index/snii/query/gram_boolean_query_test.cpp b/be/test/storage/index/snii/query/gram_boolean_query_test.cpp new file mode 100644 index 00000000000000..007820143b6f70 --- /dev/null +++ b/be/test/storage/index/snii/query/gram_boolean_query_test.cpp @@ -0,0 +1,999 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/gram_boolean_query.h" + +#include + +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "storage/index/inverted/gram/gram_query.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii_query_test_util.h" + +using namespace doris::snii; +using doris::Status; +using doris::segment_v2::gram::GramQuery; + +// R8/R24 (unity build): named rather than anonymous so these helpers cannot clash with the +// identically purposed helpers of the other query tests. +namespace gram_boolean_query_test_detail { + +// A fake GramPostingSource: a map standing in for the "gram -> sorted docid list" dictionary, so +// the AND/OR/ALL/NONE evaluation logic of gram_boolean_query can be covered without building a +// real SNII index file. The counters record how the evaluator drives the source: dictionary +// rounds (df_batches) and posting rounds (posting_batches) are what a cold object-store read pays +// for, and `lookups` still counts individual grams so the "a missing gram costs a df lookup and +// no posting read" property stays asserted. +class MapPostingSource final : public query::GramPostingSource { +public: + std::map> lists; + // Grams present in the dictionary whose posting list the writer dropped (stop-gram). + // The value is the df the entry still carries; there is no list to serve, and asking + // for one is a bug the evaluator must not commit -- hence the fatal counter below. + std::map dropped; + int lookups = 0; + int df_batches = 0; + int posting_batches = 0; + int dropped_posting_reads = 0; + + Status dfs(const std::vector& grams, std::vector* out) override { + ++df_batches; + out->assign(grams.size(), query::GramDf {}); + for (size_t i = 0; i < grams.size(); ++i) { + ++lookups; + const auto d = dropped.find(grams[i]); + if (d != dropped.end()) { + (*out)[i].found = true; + (*out)[i].df = d->second; + (*out)[i].matches_all = true; + continue; + } + auto it = lists.find(grams[i]); + if (it == lists.end()) { + continue; + } + (*out)[i].found = true; + (*out)[i].df = it->second.size(); + } + return Status::OK(); + } + + // The no-IO arm of the gate. `budget` of 0 means the fake supplies none and the + // evaluator falls back to the configured ratio, which is the pre-digest behaviour every + // other case in this file exercises. `bounded` stands in for the segment's high-df + // digest: a gram listed here has a df the source can state without reading anything. + uint64_t budget = 0; + std::map bounded; + + uint64_t candidate_budget() const override { return budget; } + + void known_dfs(const std::vector& grams, std::vector* out, + std::vector* known) const override { + out->assign(grams.size(), 0); + known->assign(grams.size(), 0); + for (size_t i = 0; i < grams.size(); ++i) { + const auto it = bounded.find(grams[i]); + if (it != bounded.end()) { + (*out)[i] = it->second; + (*known)[i] = 1; + } + } + } + + // Records an attempt to read a posting that does not exist. In production this is not + // a silent wrong answer but a refusal: the planner rejects such a term outright. + void note_if_dropped(const std::vector& grams) { + for (const std::string& gram : grams) { + if (dropped.contains(gram)) { + ++dropped_posting_reads; + } + } + } + + Status and_postings(const std::vector& grams, roaring::Roaring* out) override { + ++posting_batches; + note_if_dropped(grams); + bool seeded = false; + roaring::Roaring acc; + for (const std::string& gram : grams) { + auto it = lists.find(gram); + if (it == lists.end()) { + return Status::OK(); + } + roaring::Roaring cur; + cur.addMany(it->second.size(), it->second.data()); + if (!seeded) { + acc = std::move(cur); + seeded = true; + } else { + acc &= cur; + } + } + *out |= acc; + return Status::OK(); + } + + Status or_postings(const std::vector& grams, roaring::Roaring* out) override { + ++posting_batches; + note_if_dropped(grams); + for (const std::string& gram : grams) { + auto it = lists.find(gram); + if (it == lists.end()) { + continue; + } + out->addMany(it->second.size(), it->second.data()); + } + return Status::OK(); + } +}; + +std::vector ToVec(const roaring::Roaring& r) { + return {r.begin(), r.end()}; +} + +roaring::Roaring FullRange(uint32_t num_docs) { + roaring::Roaring full; + full.addRange(0, num_docs); + return full; +} + +// Pins BOTH knobs of the cost gate for one test: the ratio and the row floor below which the +// ratio is not applied at all. Tests that want the gate to fire have to lower the floor under +// their own segment size on purpose -- the production default (65536 rows) is far above every +// segment built here, so nothing in this file can trip the gate by accident. a bp value of 0 disables +// the gate outright, which is exactly the behaviour the evaluator had before it existed. +class ScopedCandidateRatio { +public: + ScopedCandidateRatio(int32_t bp, int32_t min_rows) + : _saved_bp(doris::config::gram_index_max_candidate_ratio_bp), + _saved_min_rows(doris::config::gram_index_candidate_ratio_min_rows) { + doris::config::gram_index_max_candidate_ratio_bp = bp; + doris::config::gram_index_candidate_ratio_min_rows = min_rows; + } + ScopedCandidateRatio(const ScopedCandidateRatio&) = delete; + ScopedCandidateRatio& operator=(const ScopedCandidateRatio&) = delete; + ScopedCandidateRatio(ScopedCandidateRatio&&) = delete; + ScopedCandidateRatio& operator=(ScopedCandidateRatio&&) = delete; + ~ScopedCandidateRatio() { + doris::config::gram_index_max_candidate_ratio_bp = _saved_bp; + doris::config::gram_index_candidate_ratio_min_rows = _saved_min_rows; + } + +private: + int32_t _saved_bp; + int32_t _saved_min_rows; +}; + +// Row floor for tests that want the gate to fire: apply the ratio at any segment size. +constexpr int32_t kGateEverySize = 1; + +GramQuery AndOf(const std::vector& grams) { + GramQuery q = GramQuery::of_gram(grams.front()); + for (size_t i = 1; i < grams.size(); ++i) { + q = GramQuery::and_(std::move(q), GramQuery::of_gram(grams[i])); + } + return q; +} + +GramQuery OrOf(const std::vector& grams) { + GramQuery q = GramQuery::of_gram(grams.front()); + for (size_t i = 1; i < grams.size(); ++i) { + q = GramQuery::or_(std::move(q), GramQuery::of_gram(grams[i])); + } + return q; +} + +// --------------------------------------------------------------------------- +// A real SNII segment for the cold-read I/O regression tests below. Every knob is set so that a +// read really costs a round trip -- SNII_DICT_RESIDENT_MAX=0 keeps the dictionary off-heap, +// target_dict_block_bytes=1 puts every gram in its own DICT block, and a 1-byte metered block +// makes any not-yet-read byte range a miss. Without that, a per-gram loop and a batched read are +// indistinguishable on a warm local file and the assertions would prove nothing. +// --------------------------------------------------------------------------- + +constexpr uint32_t kIoDocCount = 8192; +constexpr uint64_t kIoIndexId = 1; +constexpr const char* kIoIndexSuffix = "body"; + +// Six dense grams: gram i covers every doc whose bit i is clear, so each has df 4096 while their +// intersection is only the multiples of 64 -- the shape a gram AND actually meets. +std::vector DenseGrams() { + return {"gaa", "gab", "gac", "gad", "gae", "gaf"}; +} + +// Six mid-frequency grams for the OR side. +std::vector OrGrams() { + return {"rba", "rbb", "rbc", "rbd", "rbe", "rbf"}; +} + +bool DenseGramHasDoc(size_t gram_index, uint32_t docid) { + return ((docid >> gram_index) & 1U) == 0U; +} + +bool OrGramHasDoc(size_t gram_index, uint32_t docid) { + return docid % (13U + static_cast(gram_index)) == 0U; +} + +void WriteIoSegment(snii_test::MemoryFile* file) { + writer::SpimiTermBuffer buf(/*has_positions=*/false); + const std::vector dense = DenseGrams(); + const std::vector or_grams = OrGrams(); + for (uint32_t d = 0; d < kIoDocCount; ++d) { + uint32_t pos = 0; + for (size_t g = 0; g < dense.size(); ++g) { + if (DenseGramHasDoc(g, d)) { + buf.add_token(dense[g], d, pos++); + } + } + for (size_t g = 0; g < or_grams.size(); ++g) { + if (OrGramHasDoc(g, d)) { + buf.add_token(or_grams[g], d, pos++); + } + } + } + + writer::SniiIndexInput in; + in.index_id = kIoIndexId; + in.index_suffix = kIoIndexSuffix; + in.config = format::IndexConfig::kDocsOnly; + in.doc_count = kIoDocCount; + in.terms = buf.finalize_sorted(); + // One DICT block per term, so resolving k grams touches k independent cold blocks. + in.target_dict_block_bytes = 1; + + writer::SniiCompoundWriter compound(file); + snii_test::assert_ok(compound.add_logical_index(in)); + snii_test::assert_ok(compound.finish()); +} + +roaring::Roaring DenseAndTruth() { + roaring::Roaring truth; + const size_t gram_count = DenseGrams().size(); + for (uint32_t d = 0; d < kIoDocCount; ++d) { + bool all = true; + for (size_t g = 0; g < gram_count && all; ++g) { + all = DenseGramHasDoc(g, d); + } + if (all) { + truth.add(d); + } + } + return truth; +} + +roaring::Roaring OrTruth() { + roaring::Roaring truth; + const size_t gram_count = OrGrams().size(); + for (uint32_t d = 0; d < kIoDocCount; ++d) { + for (size_t g = 0; g < gram_count; ++g) { + if (OrGramHasDoc(g, d)) { + truth.add(d); + break; + } + } + } + return truth; +} + +// Serial rounds spent evaluating `q` from a cold cache. reset_metrics() clears both the counters +// and the modelled resident block set, so every measurement starts from the same cold state. +uint64_t ColdSerialRounds(io::MeteredFileReader* metered, const reader::LogicalIndexReader& idx, + const GramQuery& q, roaring::Roaring* out) { + metered->reset_metrics(); + query::LogicalIndexPostingSource source(idx); + EXPECT_TRUE(query::gram_boolean_query(source, q, kIoDocCount, out).ok()); + return metered->metrics().serial_rounds; +} + +// The reference shape: resolve and read every gram on its own, the way a one-gram-at-a-time +// posting source does. The batched evaluator must stay well below this and, more importantly, +// must not grow with the gram count. +uint64_t ColdSerialRoundsPerGram(io::MeteredFileReader* metered, + const reader::LogicalIndexReader& idx, + const std::vector& grams, bool intersect) { + metered->reset_metrics(); + for (const std::string& gram : grams) { + // A fresh source per gram models the old per-gram interface: no shared memo, so each gram + // pays its own dictionary lookup plus its own posting read. + query::LogicalIndexPostingSource source(idx); + const std::vector single = {gram}; + std::vector dfs; + EXPECT_TRUE(source.dfs(single, &dfs).ok()); + EXPECT_EQ(dfs.size(), single.size()); + roaring::Roaring cur; + EXPECT_TRUE(intersect ? source.and_postings(single, &cur).ok() + : source.or_postings(single, &cur).ok()); + } + return metered->metrics().serial_rounds; +} + +} // namespace gram_boolean_query_test_detail + +using gram_boolean_query_test_detail::AndOf; +using gram_boolean_query_test_detail::ColdSerialRounds; +using gram_boolean_query_test_detail::ColdSerialRoundsPerGram; +using gram_boolean_query_test_detail::DenseAndTruth; +using gram_boolean_query_test_detail::DenseGrams; +using gram_boolean_query_test_detail::FullRange; +using gram_boolean_query_test_detail::kGateEverySize; +using gram_boolean_query_test_detail::kIoDocCount; +using gram_boolean_query_test_detail::kIoIndexId; +using gram_boolean_query_test_detail::kIoIndexSuffix; +using gram_boolean_query_test_detail::MapPostingSource; +using gram_boolean_query_test_detail::OrGrams; +using gram_boolean_query_test_detail::OrOf; +using gram_boolean_query_test_detail::OrTruth; +using gram_boolean_query_test_detail::ScopedCandidateRatio; +using gram_boolean_query_test_detail::ToVec; +using gram_boolean_query_test_detail::WriteIoSegment; + +// The evaluation tests below run on the PRODUCTION defaults on purpose: a ten-doc query is +// orders of magnitude under config::gram_index_candidate_ratio_min_rows, so the cost gate is +// structurally unable to fire and these assertions cover the default configuration path. Only +// the gate's own tests lower the row floor to make it fire. +TEST(GramBooleanQueryTest, AndOrAllNone) { + MapPostingSource src; + src.lists["abc"] = {1, 2, 3, 7}; + src.lists["bcd"] = {2, 3, 9}; + src.lists["xyz"] = {7}; + + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query( + src, GramQuery::and_(GramQuery::of_gram("abc"), GramQuery::of_gram("bcd")), + 10, &out) + .ok()); + EXPECT_EQ(ToVec(out), (std::vector {2, 3})); + + out = roaring::Roaring(); + ASSERT_TRUE(query::gram_boolean_query( + src, GramQuery::or_(GramQuery::of_gram("bcd"), GramQuery::of_gram("xyz")), + 10, &out) + .ok()); + EXPECT_EQ(ToVec(out), (std::vector {2, 3, 7, 9})); + + out = roaring::Roaring(); + ASSERT_TRUE(query::gram_boolean_query(src, GramQuery::all(), 4, &out).ok()); + EXPECT_EQ(ToVec(out), (std::vector {0, 1, 2, 3})); + + out = roaring::Roaring(); + ASSERT_TRUE(query::gram_boolean_query(src, GramQuery::none(), 4, &out).ok()); + EXPECT_TRUE(out.isEmpty()); +} + +TEST(GramBooleanQueryTest, MissingGramIsNoneAndEarlyExit) { + MapPostingSource src; + src.lists["abc"] = {1, 2, 3}; + + roaring::Roaring out; + auto q = GramQuery::and_(GramQuery::of_gram("abc"), GramQuery::of_gram("nope")); + ASSERT_TRUE(query::gram_boolean_query(src, q, 10, &out).ok()); + EXPECT_TRUE(out.isEmpty()); + // A missing gram makes the whole AND empty after a df lookup alone, so no posting should be + // read: one df lookup per gram, resolved in a single dictionary round, and no posting round. + EXPECT_EQ(src.lookups, 2); + EXPECT_EQ(src.df_batches, 1); + EXPECT_EQ(src.posting_batches, 0); +} + +TEST(GramBooleanQueryTest, NestedAndOfOr) { + MapPostingSource src; + src.lists["a"] = {1, 2, 3, 4}; + src.lists["b"] = {2}; + src.lists["c"] = {4, 5}; + + roaring::Roaring out; + auto q = GramQuery::and_(GramQuery::of_gram("a"), + GramQuery::or_(GramQuery::of_gram("b"), GramQuery::of_gram("c"))); + ASSERT_TRUE(query::gram_boolean_query(src, q, 10, &out).ok()); + EXPECT_EQ(ToVec(out), (std::vector {2, 4})); +} + +// Each query node must cost ONE dictionary round and ONE posting round no matter how many grams +// it holds. On object storage that is the difference between a constant and a linear number of +// remote round trips, so it is asserted on the fake source too, where it is exact. +TEST(GramBooleanQueryTest, EachNodeCostsOneDictAndOnePostingRound) { + MapPostingSource src; + src.lists["a"] = {1, 2, 3, 4, 5}; + src.lists["b"] = {2, 3, 4, 5}; + src.lists["c"] = {3, 4, 5}; + src.lists["d"] = {4, 5}; + + roaring::Roaring out; + auto q = GramQuery::and_(GramQuery::and_(GramQuery::of_gram("a"), GramQuery::of_gram("b")), + GramQuery::and_(GramQuery::of_gram("c"), GramQuery::of_gram("d"))); + ASSERT_TRUE(query::gram_boolean_query(src, q, 10, &out).ok()); + EXPECT_EQ(ToVec(out), (std::vector {4, 5})); + // and_() flattens the nested ANDs into a single four-gram node. + EXPECT_EQ(src.df_batches, 1); + EXPECT_EQ(src.posting_batches, 1); + EXPECT_EQ(src.lookups, 4); + + src.df_batches = 0; + src.posting_batches = 0; + src.lookups = 0; + out = roaring::Roaring(); + auto or_q = GramQuery::or_(GramQuery::or_(GramQuery::of_gram("a"), GramQuery::of_gram("b")), + GramQuery::or_(GramQuery::of_gram("c"), GramQuery::of_gram("d"))); + ASSERT_TRUE(query::gram_boolean_query(src, or_q, 10, &out).ok()); + EXPECT_EQ(ToVec(out), (std::vector {1, 2, 3, 4, 5})); + EXPECT_EQ(src.df_batches, 1); + EXPECT_EQ(src.posting_batches, 1); + EXPECT_EQ(src.lookups, 4); +} + +// A single-gram AND has an exact candidate count before any posting read, so the gate fires on +// df alone and the posting read is skipped entirely. +TEST(GramBooleanQueryTest, CandidateRatioGateSkipsPostingReadForSingleGram) { + MapPostingSource src; + src.lists["hot"] = {0, 1, 2, 3, 4, 5, 6, 7}; // df 8 of 10 docs + + { + ScopedCandidateRatio ratio(3000, kGateEverySize); + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, GramQuery::of_gram("hot"), 10, &out).ok()); + // Giving up means "prune nothing", i.e. the whole docid space, never a smaller set. + EXPECT_TRUE(out == FullRange(10)); + EXPECT_EQ(src.df_batches, 1); + EXPECT_EQ(src.posting_batches, 0) << "the df bound alone must decide, before any IO"; + } + + src.df_batches = 0; + src.posting_batches = 0; + { + // Gate disabled: exactly the pre-gate behaviour. + ScopedCandidateRatio ratio(0, kGateEverySize); + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, GramQuery::of_gram("hot"), 10, &out).ok()); + EXPECT_EQ(ToVec(out), (std::vector {0, 1, 2, 3, 4, 5, 6, 7})); + EXPECT_EQ(src.posting_batches, 1); + } +} + +// A union is at least as large as its largest member, so max(df) gates an OR node before any +// posting -- or any sub-query -- is touched. +TEST(GramBooleanQueryTest, CandidateRatioGateSkipsPostingReadForOr) { + MapPostingSource src; + src.lists["hot"] = {0, 1, 2, 3, 4, 5}; // df 6 of 10 docs + src.lists["cold"] = {9}; + + { + ScopedCandidateRatio ratio(3000, kGateEverySize); + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query( + src, + GramQuery::or_(GramQuery::of_gram("hot"), GramQuery::of_gram("cold")), + 10, &out) + .ok()); + EXPECT_TRUE(out == FullRange(10)); + EXPECT_EQ(src.df_batches, 1); + EXPECT_EQ(src.posting_batches, 0); + } + + src.df_batches = 0; + src.posting_batches = 0; + { + ScopedCandidateRatio ratio(7000, kGateEverySize); + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query( + src, + GramQuery::or_(GramQuery::of_gram("hot"), GramQuery::of_gram("cold")), + 10, &out) + .ok()); + // 7 of 10 candidates is not above 70%, so the gate stays out of the way. + EXPECT_EQ(ToVec(out), (std::vector {0, 1, 2, 3, 4, 5, 9})); + EXPECT_EQ(src.posting_batches, 1); + } +} + +// A multi-gram AND is gated on min(df), before a single posting is read. min(df) bounds the node +// from above -- an intersection can never hold more docs than its rarest gram -- so a min(df) +// inside the budget lets the node through, and one above it means the node is only worth reading +// if the intersection collapses far below its rarest gram. Giving up there is deliberately +// pessimistic: measured against remote object storage, a node whose candidates stay above the +// budget cannot skip a single page of the column, so its posting reads are pure overhead. +// +// Whichever arm fires, giving up yields the full range, which is a superset of what the ungated +// evaluation returns -- not one matching row can be lost. +TEST(GramBooleanQueryTest, CandidateRatioGatePrejudgesMultiGramAndOnMinDf) { + MapPostingSource src; + src.lists["a"] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + src.lists["b"] = {0, 1, 2, 3, 4, 5, 6, 7, 9}; + + roaring::Roaring gated; + { + // Both grams are far above the ratio, so min(df) is too, and the node is abandoned on the + // df batch alone. + ScopedCandidateRatio ratio(3000, kGateEverySize); + ASSERT_TRUE(query::gram_boolean_query( + src, GramQuery::and_(GramQuery::of_gram("a"), GramQuery::of_gram("b")), + 10, &gated) + .ok()); + EXPECT_EQ(src.posting_batches, 0) + << "the gate must fire on the df batch, before paying for any posting read"; + EXPECT_TRUE(gated == FullRange(10)) << "giving up widens, it never narrows"; + } + + src.posting_batches = 0; + roaring::Roaring ungated; + { + // Same node, gate off: the true candidate set. Every row of it is inside the gated result + // above, which is what "giving up never drops a row" means. + ScopedCandidateRatio ratio(0, kGateEverySize); + ASSERT_TRUE(query::gram_boolean_query( + src, GramQuery::and_(GramQuery::of_gram("a"), GramQuery::of_gram("b")), + 10, &ungated) + .ok()); + EXPECT_EQ(src.posting_batches, 1); + EXPECT_EQ(ToVec(ungated), (std::vector {0, 1, 2, 3, 4, 5, 6, 7})); + } + EXPECT_TRUE((ungated - gated).isEmpty()) + << "the gated result must contain every true candidate"; +} + +// The safe direction of the same rule: one rare gram is enough to carry a node through, however +// common its neighbours are. This is what keeps the gate from throwing away the selective patterns +// the index exists for -- a pattern worth an index is selective through its rarest gram. +TEST(GramBooleanQueryTest, CandidateRatioGateLetsARareGramCarryTheNode) { + MapPostingSource src; + src.lists["common"] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + src.lists["rare"] = {3}; + + ScopedCandidateRatio ratio(3000, kGateEverySize); + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query( + src, + GramQuery::and_(GramQuery::of_gram("common"), GramQuery::of_gram("rare")), + 10, &out) + .ok()); + EXPECT_EQ(src.posting_batches, 1) << "min(df) is inside the budget, so the node must proceed"; + EXPECT_EQ(ToVec(out), (std::vector {3})); +} + +// The shape min(df) cannot see, and the one that made a CJK column's index worthless. +// +// min(df) bounds an AND from above, and for grams cut out of one ASCII literal that bound is +// close to the truth: the grams overlap almost completely, so the intersection really is about +// as big as the rarest of them. Nothing forces that. A CJK column indexes one term per code +// point, so a four-character literal is an AND of four ordinary characters -- each common on +// its own, sharing almost no documents beyond the rows that spell the literal. Measured on +// weibo: one four-character literal has 18 matching rows and its AND returns 28 candidates, +// while another has 5 and was abandoned outright, because its rarest character is still a +// common one. +// +// Below, four grams each cover 20% of the segment and their intersection is a single document. +// A gate reading min(df) sees 20%, exceeds a 15% budget and gives up on a node that returns one +// row. +TEST(GramBooleanQueryTest, TheGateReadsANodeWhoseCommonGramsIntersectRarely) { + constexpr uint32_t kDocs = 2000; + constexpr uint32_t kBlock = 399; + MapPostingSource src; + const char* names[4] = {"g0", "g1", "g2", "g3"}; + for (uint32_t g = 0; g < 4; g++) { + std::vector docs {0}; // the one document every gram shares + for (uint32_t d = 0; d < kBlock; d++) { + docs.push_back(1 + g * kBlock + d); // disjoint from every other gram's block + } + src.lists[names[g]] = docs; + } + + ScopedCandidateRatio ratio(1500, kGateEverySize); // 15%; each gram alone is 20% + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query( + src, + GramQuery::and_(GramQuery::and_(GramQuery::and_(GramQuery::of_gram("g0"), + GramQuery::of_gram("g1")), + GramQuery::of_gram("g2")), + GramQuery::of_gram("g3")), + kDocs, &out) + .ok()); + EXPECT_EQ(src.posting_batches, 1) + << "grams that are common individually can still intersect rarely; the gate must " + "judge the intersection, not its largest possible size"; + EXPECT_EQ(ToVec(out), (std::vector {0})); +} + +// The same mistake one level up. `(A|B)C` compiles to an AND whose direct gram leaves are the +// grams of C and whose sub-query is the alternation, and the gate judged such a node on the +// leaves alone. Measured on weibo with a four-character CJK literal `XY` (a two-character +// brand X followed by a two-character noun Y): `XY` returns 148 candidates for its 18 rows, +// while `(X|X')Y` -- 23 rows, a strictly smaller answer -- was abandoned with the whole +// segment as candidates, because the two characters of Y are common and the alternation +// that made the node selective was not looked at. +// +// A sub-query can only narrow an AND, and once evaluated its cardinality is an exact upper +// bound on the whole node, so the node below must be read however common its own leaves are. +TEST(GramBooleanQueryTest, TheGateCountsWhatASubQueryAlreadyNarrowedTo) { + constexpr uint32_t kDocs = 2000; + MapPostingSource src; + // Two leaves covering half the segment each: on their own, far over the budget. + std::vector half; + for (uint32_t d = 0; d < kDocs / 2; ++d) { + half.push_back(d); + } + src.lists["common_x"] = half; + src.lists["common_y"] = half; + // The alternation is what makes the node selective: three documents in total, one of them + // outside the common leaves so the leaves are shown to still take part in the answer. + src.lists["rare_a"] = {1, 2}; + src.lists["rare_b"] = {1500}; + + ScopedCandidateRatio ratio(1500, kGateEverySize); // 15%; each leaf alone is 50% + roaring::Roaring out; + const GramQuery node = GramQuery::and_( + AndOf({"common_x", "common_y"}), + GramQuery::or_(GramQuery::of_gram("rare_a"), GramQuery::of_gram("rare_b"))); + ASSERT_TRUE(query::gram_boolean_query(src, node, kDocs, &out).ok()); + EXPECT_EQ(ToVec(out), (std::vector {1, 2})) + << "the sub-query narrows the node to three documents, two of which hold the common " + "leaves as well; abandoning it on the leaves alone throws that away"; +} + +// Every defect this gate has had looked the same from outside: the index was consulted and +// eliminated nothing. A profile could only show that indirectly -- a candidate count equal to +// the segment with zero rows filtered -- which a reader has to already suspect to go looking +// for. Reporting the decision makes "the index gave up here" a fact rather than an inference. +TEST(GramBooleanQuery, TheGateReportsWhenItGivesUp) { + MapPostingSource src; + src.lists["a"] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + src.lists["b"] = {0, 1, 2, 3, 4, 5, 6, 7, 9}; + src.lists["rare"] = {3}; + + ScopedCandidateRatio ratio(3000, kGateEverySize); + { + query::GramGateStats stats; + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, AndOf({"a", "b"}), 10, &out, &stats).ok()); + EXPECT_TRUE(out == FullRange(10)); + EXPECT_EQ(stats.nodes_given_up, 1U) << "the node the gate abandoned must be reported"; + } + { + // One rare gram carries the node, so nothing is abandoned and nothing is reported. + query::GramGateStats stats; + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, AndOf({"a", "rare"}), 10, &out, &stats).ok()); + EXPECT_EQ(ToVec(out), (std::vector {3})); + EXPECT_EQ(stats.nodes_given_up, 0U) << "a node that was read must not be reported"; + } +} + +// --------------------------------------------------------------------------- +// I/O regression: the number of serial (cold) read rounds a gram node costs must be a small +// constant, not a function of the gram count. Without these assertions the batching added here +// can silently regress back into a per-gram loop, which is invisible on a warm local file and +// fatal on object storage. +// --------------------------------------------------------------------------- + +TEST(GramBooleanQueryIoTest, AndBatchesDictionaryAndPostingReads) { + // Runs on the production gate defaults. This segment is below the configured row floor, so + // the gate cannot fire and the full read path really is measured; assert that rather than + // assume it, so lowering the default floor cannot silently turn this into a no-op. + ASSERT_LT(kIoDocCount, + static_cast(doris::config::gram_index_candidate_ratio_min_rows)); + snii_test::ScopedEnv on_demand_dict("SNII_DICT_RESIDENT_MAX", "0"); + + snii_test::MemoryFile file; + WriteIoSegment(&file); + if (::testing::Test::HasFatalFailure()) { + return; + } + + io::MeteredFileReader metered(&file, /*block_size=*/1); + reader::SniiSegmentReader segment; + snii_test::assert_ok(reader::SniiSegmentReader::open(&metered, &segment)); + reader::LogicalIndexReader idx; + snii_test::assert_ok(segment.open_index(kIoIndexId, kIoIndexSuffix, &idx)); + + const std::vector grams = DenseGrams(); + roaring::Roaring got; + const uint64_t batched = ColdSerialRounds(&metered, idx, AndOf(grams), &got); + EXPECT_TRUE(got == DenseAndTruth()); + + // The same node with half the grams: what the extra three grams cost tells batching apart + // from a per-gram loop far more reliably than any absolute number. + const std::vector half(grams.begin(), grams.begin() + 3); + roaring::Roaring got_half; + const uint64_t batched_half = ColdSerialRounds(&metered, idx, AndOf(half), &got_half); + + const uint64_t per_gram = ColdSerialRoundsPerGram(&metered, idx, grams, /*intersect=*/true); + EXPECT_GT(per_gram, batched) << "batching must beat the one-gram-at-a-time shape"; + // Round budget for a k-gram AND: ONE batched dictionary wave + ONE batched prelude round + + // at most one docid round per gram. That last term is the leapfrog conjunction shared with + // boolean_and(): it is deliberately sequential, because each gram is only read over the + // windows the grams before it left alive. The two fixed stages are what must never become + // per-gram again -- a per-gram loop measures three rounds per gram (see per_gram). + EXPECT_LE(batched, grams.size() + 2) + << "a gram AND must pay ONE dictionary round and ONE prelude round for all grams " + "together (per-gram reference was " + << per_gram << ")"; + EXPECT_LE(batched - batched_half, grams.size() - half.size()) + << "each extra gram may add at most the one conjunction round it is read in, never " + "its own dictionary and prelude rounds too"; +} + +TEST(GramBooleanQueryIoTest, OrBatchesDictionaryAndPostingReads) { + // Production gate defaults; see AndBatchesDictionaryAndPostingReads for why the floor is + // asserted rather than assumed. + ASSERT_LT(kIoDocCount, + static_cast(doris::config::gram_index_candidate_ratio_min_rows)); + snii_test::ScopedEnv on_demand_dict("SNII_DICT_RESIDENT_MAX", "0"); + + snii_test::MemoryFile file; + WriteIoSegment(&file); + if (::testing::Test::HasFatalFailure()) { + return; + } + + io::MeteredFileReader metered(&file, /*block_size=*/1); + reader::SniiSegmentReader segment; + snii_test::assert_ok(reader::SniiSegmentReader::open(&metered, &segment)); + reader::LogicalIndexReader idx; + snii_test::assert_ok(segment.open_index(kIoIndexId, kIoIndexSuffix, &idx)); + + const std::vector grams = OrGrams(); + roaring::Roaring got; + const uint64_t batched = ColdSerialRounds(&metered, idx, OrOf(grams), &got); + EXPECT_TRUE(got == OrTruth()); + + const std::vector half(grams.begin(), grams.begin() + 3); + roaring::Roaring got_half; + const uint64_t batched_half = ColdSerialRounds(&metered, idx, OrOf(half), &got_half); + + const uint64_t per_gram = ColdSerialRoundsPerGram(&metered, idx, grams, /*intersect=*/false); + EXPECT_GT(per_gram, batched) << "batching must beat the one-gram-at-a-time shape"; + // A union has nothing to leapfrog, so it is fully batched: ONE dictionary round and ONE + // posting round, whatever the gram count. + EXPECT_EQ(batched, batched_half) + << "a gram OR must not cost more rounds for six grams than for three (per-gram " + "reference was " + << per_gram << ")"; + EXPECT_LE(batched, 3U) << "a gram OR must cost a constant, small number of serial rounds"; +} + +// The gate exists to save exactly this: when the df bound already says the node cannot prune, the +// posting read never happens, and the result is the whole segment. Both knobs are pinned: this +// 8192-row segment is under the production row floor, so the gate is made to fire on purpose +// rather than by accident. +TEST(GramBooleanQueryIoTest, CandidateRatioGateSkipsPostingIo) { + snii_test::ScopedEnv on_demand_dict("SNII_DICT_RESIDENT_MAX", "0"); + + snii_test::MemoryFile file; + WriteIoSegment(&file); + if (::testing::Test::HasFatalFailure()) { + return; + } + + io::MeteredFileReader metered(&file, /*block_size=*/1); + reader::SniiSegmentReader segment; + snii_test::assert_ok(reader::SniiSegmentReader::open(&metered, &segment)); + reader::LogicalIndexReader idx; + snii_test::assert_ok(segment.open_index(kIoIndexId, kIoIndexSuffix, &idx)); + + // One dense gram: df is 50% of the segment, so a 30% gate must give up on the df alone. + const GramQuery q = GramQuery::of_gram(DenseGrams().front()); + + roaring::Roaring ungated; + uint64_t ungated_rounds = 0; + { + ScopedCandidateRatio ratio(0, kGateEverySize); + ungated_rounds = ColdSerialRounds(&metered, idx, q, &ungated); + } + ASSERT_FALSE(ungated.isEmpty()); + + roaring::Roaring gated; + uint64_t gated_rounds = 0; + { + ScopedCandidateRatio ratio(3000, kGateEverySize); + gated_rounds = ColdSerialRounds(&metered, idx, q, &gated); + } + EXPECT_TRUE(gated == FullRange(kIoDocCount)) << "giving up must widen to the whole segment"; + EXPECT_TRUE((ungated - gated).isEmpty()) << "giving up must never drop a candidate row"; + EXPECT_LT(gated_rounds, ungated_rounds) << "the gate must actually save the posting read"; +} + +// --------------------------------------------------------------------------- +// stop-gram: a gram present in the dictionary whose posting list the writer dropped. +// +// It matches every document, so it constrains nothing. The whole risk of the feature lives +// in one confusion: a dropped gram and an absent gram are both "no posting list to read", +// and they mean opposite things. Absent empties an AND; dropped leaves it untouched. These +// cases pin both directions, and every one of them also asserts that the evaluator never +// asks for the posting list that does not exist. +// --------------------------------------------------------------------------- + +TEST(GramBooleanQuery, StopGramDropsOutOfAnAndWithoutChangingIt) { + MapPostingSource src; + src.lists["abc"] = {3, 7, 11, 40}; + src.lists["bcd"] = {7, 11, 90}; + src.dropped["the"] = 900; // in the dictionary, no posting list + + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, AndOf({"abc", "the", "bcd"}), 1000, &out).ok()); + // Identical to the same AND without the stop-gram: X AND ALL == X. + EXPECT_EQ(ToVec(out), (std::vector {7, 11})); + EXPECT_EQ(src.dropped_posting_reads, 0) << "a dropped posting must never be read"; +} + +TEST(GramBooleanQuery, AnAndOfOnlyStopGramsIsAll) { + MapPostingSource src; + src.dropped["the"] = 900; + src.dropped["and"] = 850; + + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, AndOf({"the", "and"}), 1000, &out).ok()); + EXPECT_TRUE(out == FullRange(1000)) << "grams that constrain nothing leave the node at ALL"; + EXPECT_EQ(src.posting_batches, 0) << "nothing to read: no posting round at all"; + EXPECT_EQ(src.dropped_posting_reads, 0); +} + +// The direction that matters most: dropped must not be treated as absent, which would +// empty the node and lose every matching row. +TEST(GramBooleanQuery, StopGramIsNotTreatedAsAMissingGram) { + MapPostingSource present; + present.lists["abc"] = {1, 2, 3}; + present.dropped["the"] = 900; + roaring::Roaring dropped_out; + ASSERT_TRUE(query::gram_boolean_query(present, AndOf({"abc", "the"}), 1000, &dropped_out).ok()); + EXPECT_EQ(ToVec(dropped_out), (std::vector {1, 2, 3})); + + // Same query shape, but the second gram is genuinely absent: that IS empty. + MapPostingSource absent; + absent.lists["abc"] = {1, 2, 3}; + roaring::Roaring absent_out; + ASSERT_TRUE(query::gram_boolean_query(absent, AndOf({"abc", "zzz"}), 1000, &absent_out).ok()); + EXPECT_TRUE(absent_out.isEmpty()) << "an absent gram still empties the AND"; +} + +TEST(GramBooleanQuery, AnOrHoldingAStopGramIsAll) { + MapPostingSource src; + src.lists["abc"] = {1, 2}; + src.dropped["the"] = 900; + + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, OrOf({"abc", "the"}), 1000, &out).ok()); + EXPECT_TRUE(out == FullRange(1000)) << "a union with a match-all member is everything"; + EXPECT_EQ(src.posting_batches, 0) << "the answer is known from the df batch alone"; + EXPECT_EQ(src.dropped_posting_reads, 0); +} + +// A stop-gram inside an AND must not suppress the sub-queries, which still narrow the node. +TEST(GramBooleanQuery, StopGramLeavesSubQueriesInAnAndIntact) { + MapPostingSource src; + src.lists["abc"] = {1, 2, 3, 4}; + src.lists["bcd"] = {3, 4, 5}; + src.dropped["the"] = 900; + + GramQuery q = GramQuery::and_(GramQuery::of_gram("the"), OrOf({"abc", "bcd"})); + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, q, 1000, &out).ok()); + EXPECT_EQ(ToVec(out), (std::vector {1, 2, 3, 4, 5})); + EXPECT_EQ(src.dropped_posting_reads, 0); +} + +// The gate and the stop-gram meet on the same node: the surviving grams decide it, because +// a term that matches everything cannot be the rarest one and must not stand in for it. +TEST(GramBooleanQuery, TheGateJudgesAnAndOnItsConstrainingGramsOnly) { + ScopedCandidateRatio ratio(/*bp=*/1000, kGateEverySize); // give up above 10% of the segment + MapPostingSource src; + src.lists["abc"] = {1, 2, 3}; // 0.3% of 1000 docs: well inside the budget + src.dropped["the"] = 999; // would blow any budget if it were counted + + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, AndOf({"abc", "the"}), 1000, &out).ok()); + EXPECT_EQ(ToVec(out), (std::vector {1, 2, 3})) + << "the rare gram still carries the node"; + EXPECT_EQ(src.dropped_posting_reads, 0); +} + +// --------------------------------------------------------------------------- +// The no-IO arm of the cost gate. +// +// Reading a df costs a remote dictionary round trip -- measured at 40 seconds across a +// node's grams on a segment whose cache holds nothing -- and the gate often spends it only +// to conclude the node is not worth pruning. When the segment can bound those df values +// from its own resident digest, that conclusion is free. The direction of the bound is the +// whole correctness argument, and it differs between AND and OR. +// --------------------------------------------------------------------------- + +TEST(GramBooleanQuery, AnAndOverBudgetOnKnownDfsGivesUpWithoutReadingAnything) { + MapPostingSource src; + src.lists["abc"] = {1, 2, 3}; + src.lists["bcd"] = {2, 3, 4}; + src.budget = 100; + src.bounded["abc"] = 500; // both known, both far over budget + src.bounded["bcd"] = 900; + + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, AndOf({"abc", "bcd"}), 10000, &out).ok()); + EXPECT_TRUE(out == FullRange(10000)) << "giving up must widen to the whole segment"; + EXPECT_EQ(src.df_batches, 0) << "the point is that no dictionary read happens at all"; + EXPECT_EQ(src.posting_batches, 0); +} + +// The asymmetry that makes it correct: an AND is estimated from all its dfs, and the digest +// holds only the segment's common terms, so a gram it cannot state is a rare one -- possibly +// the one carrying the node. It must be read rather than assumed common. +TEST(GramBooleanQuery, AnAndWithOneUnknownDfStillReadsTheDictionary) { + MapPostingSource src; + src.lists["abc"] = {1, 2, 3}; + src.lists["rare"] = {7}; + src.budget = 100; + src.bounded["abc"] = 5000; // known and over budget + // "rare" is absent from the digest: unknown, and in fact rare enough to carry the node. + + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, AndOf({"abc", "rare"}), 10000, &out).ok()); + EXPECT_EQ(src.df_batches, 1) << "an unknown gram must be looked up, not assumed common"; + EXPECT_TRUE(out.isEmpty() || out.cardinality() <= 3) + << "the node is decided by the rare gram, not widened to ALL"; +} + +// An OR is the mirror: it is at least as large as any one member, so a single known gram +// over budget settles it without the others being known at all. +TEST(GramBooleanQuery, AnOrGivesUpOnASingleKnownDfOverBudget) { + MapPostingSource src; + src.lists["abc"] = {1, 2}; + src.lists["unknown_one"] = {5}; + src.budget = 100; + src.bounded["abc"] = 4000; // only this one is known + + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, OrOf({"abc", "unknown_one"}), 10000, &out).ok()); + EXPECT_TRUE(out == FullRange(10000)); + EXPECT_EQ(src.df_batches, 0) << "one known member over budget is enough"; +} + +// Known and under budget must not trigger anything: the node proceeds normally. +TEST(GramBooleanQuery, KnownDfsUnderBudgetDoNotGiveUp) { + MapPostingSource src; + src.lists["abc"] = {1, 2, 3}; + src.lists["bcd"] = {2, 3, 9}; + src.budget = 1000; + src.bounded["abc"] = 3; + src.bounded["bcd"] = 3; + + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, AndOf({"abc", "bcd"}), 10000, &out).ok()); + EXPECT_EQ(ToVec(out), (std::vector {2, 3})); + EXPECT_EQ(src.df_batches, 1) << "the node was worth reading, so it was read"; +} + +// A source with no digest (a segment written before it existed) must behave exactly as +// before: the configured ratio decides, and nothing is skipped on bounds that do not exist. +TEST(GramBooleanQuery, WithoutABudgetTheConfiguredRatioStillGoverns) { + ScopedCandidateRatio ratio(/*bp=*/1000, kGateEverySize); + MapPostingSource src; + src.lists["abc"] = std::vector(5000); + for (uint32_t i = 0; i < 5000; ++i) { + src.lists["abc"][i] = i; + } + src.budget = 0; // no digest + + roaring::Roaring out; + ASSERT_TRUE(query::gram_boolean_query(src, AndOf({"abc"}), 10000, &out).ok()); + EXPECT_TRUE(out == FullRange(10000)) << "50% of the segment is far above a 10% ratio"; + EXPECT_EQ(src.df_batches, 1) << "without a digest the df must be read to decide"; +} diff --git a/be/test/storage/index/snii/snii_gram_cache_test.cpp b/be/test/storage/index/snii/snii_gram_cache_test.cpp new file mode 100644 index 00000000000000..f20b722ae1712e --- /dev/null +++ b/be/test/storage/index/snii/snii_gram_cache_test.cpp @@ -0,0 +1,522 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/field.h" +#include "gen_cpp/AgentService_types.h" +#include "io/fs/local_file_system.h" +#include "runtime/exec_env.h" +#include "runtime/index_policy/index_policy_mgr.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/index_query_context.h" +#include "storage/index/inverted/inverted_index_cache.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/snii_index_reader.h" +#include "storage/index/snii/snii_index_writer.h" +#include "storage/olap_common.h" +#include "storage/tablet/tablet_schema.h" +#include "util/slice.h" + +namespace doris::segment_v2 { +namespace { + +constexpr const char* kTestDir = "./ut_dir/snii_gram_cache_test"; +constexpr const char* kPathPrefix = "./ut_dir/snii_gram_cache_test/segment"; +constexpr size_t kConsumers = 8; + +// Hold the actual bitmap-computation branch until every expected consumer has +// joined. The timeout diagnoses a missing branch; it never chooses the schedule. +class GramFlightGate { +public: + static void leader(void* opaque) noexcept { + auto& gate = *static_cast(opaque); + std::unique_lock lock(gate._mutex); + ++gate._leaders; + gate._cv.notify_all(); + gate._cv.wait(lock, [&] { return gate._released; }); + } + + static void follower(void* opaque) noexcept { + auto& gate = *static_cast(opaque); + std::lock_guard lock(gate._mutex); + ++gate._followers; + gate._cv.notify_all(); + } + + bool wait_for_consumers(size_t leaders, size_t followers) { + std::unique_lock lock(_mutex); + return _cv.wait_for(lock, std::chrono::seconds(10), + [&] { return _leaders == leaders && _followers == followers; }); + } + + void release() { + std::lock_guard lock(_mutex); + _released = true; + _cv.notify_all(); + } + + // Read only after every worker has joined. + size_t leaders() const { return _leaders; } + size_t followers() const { return _followers; } + +private: + std::mutex _mutex; + std::condition_variable _cv; + size_t _leaders = 0; + size_t _followers = 0; + bool _released = false; +}; + +struct GramCacheRequest { + std::shared_ptr reader; + std::string pattern; + InvertedIndexQueryType type = InvertedIndexQueryType::LIKE_GRAM_QUERY; + bool query_cache = true; + bool searcher_cache = true; +}; + +struct GramCacheResult { + Status status; + std::shared_ptr bitmap; + OlapReaderStatistics stats; +}; + +class SniiGramCacheTest : public testing::Test { +protected: + void SetUp() override { + auto* exec_env = ExecEnv::GetInstance(); + _previous_policy_mgr = exec_env->index_policy_mgr(); + _previous_query_cache = exec_env->get_inverted_index_query_cache(); + _previous_searcher_cache = exec_env->get_inverted_index_searcher_cache(); + exec_env->_index_policy_mgr = &_policy_mgr; + _query_cache.reset(InvertedIndexQueryCache::create_global_cache(1024 * 1024, 1)); + _searcher_cache.reset(InvertedIndexSearcherCache::create_global_instance(1024 * 1024, 1)); + exec_env->set_inverted_index_query_cache(_query_cache.get()); + exec_env->set_inverted_index_searcher_cache(_searcher_cache.get()); + _worker_tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::QUERY, + "SniiGramCacheTest"); + + for (size_t i = 0; i < _indexes.size(); ++i) { + const std::string min_gram = std::to_string(3 + i); + TIndexPolicy tokenizer; + tokenizer.id = 6753900 + 2 * i; + tokenizer.name = "gram_cache_dense" + min_gram + "_tokenizer"; + tokenizer.type = TIndexPolicyType::TOKENIZER; + tokenizer.properties = {{"type", "ngram"}, {"mode", "dense"}, {"min_gram", min_gram}}; + TIndexPolicy analyzer; + analyzer.id = tokenizer.id + 1; + analyzer.name = "gram_cache_dense" + min_gram + "_analyzer"; + analyzer.type = TIndexPolicyType::ANALYZER; + analyzer.properties = {{"tokenizer", tokenizer.name}}; + _policy_mgr.apply_policy_changes({tokenizer, analyzer}, {}); + + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(6753910 + i); + pb.set_index_name("gram_cache_dense" + min_gram); + pb.add_col_unique_id(0); + pb.mutable_properties()->insert({"analyzer", analyzer.name}); + _indexes[i].init_from_pb(pb); + } + + ASSERT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + ASSERT_TRUE(io::global_local_filesystem()->create_directory(kTestDir).ok()); + auto status = write_indexes(); + ASSERT_TRUE(status.ok()) << status; + _file_reader = std::make_shared(io::global_local_filesystem(), kPathPrefix, + InvertedIndexStorageFormatPB::SNII); + status = _file_reader->init(); + ASSERT_TRUE(status.ok()) << status; + for (size_t i = 0; i < _readers.size(); ++i) { + _readers[i] = SniiIndexReader::create_shared(&_indexes[i], _file_reader, + InvertedIndexReaderType::FULLTEXT, + _values.size(), /*column_is_array=*/false); + _readers[i]->set_searcher_open_observer_for_test( + [](void* opaque) noexcept { + static_cast*>(opaque)->fetch_add(1); + }, + &_searcher_opens); + } + } + + void TearDown() override { + _readers = {}; + auto* exec_env = ExecEnv::GetInstance(); + exec_env->set_inverted_index_query_cache(_previous_query_cache); + exec_env->set_inverted_index_searcher_cache(_previous_searcher_cache); + _query_cache.reset(); + _searcher_cache.reset(); + _file_reader.reset(); + exec_env->_index_policy_mgr = _previous_policy_mgr; + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + } + + Status write_indexes() { + io::FileWriterPtr file_writer; + RETURN_IF_ERROR(io::global_local_filesystem()->create_file( + std::string(kPathPrefix) + ".idx", &file_writer)); + IndexFileWriter index_writer(io::global_local_filesystem(), kPathPrefix, + "gram_cache_rowset", 0, InvertedIndexStorageFormatPB::SNII, + std::move(file_writer), true, 6753910); + std::vector slices; + for (const auto& value : _values) { + slices.emplace_back(value); + } + for (const auto& index : _indexes) { + SniiIndexColumnWriter writer(&index_writer, &index, FieldType::OLAP_FIELD_TYPE_VARCHAR); + RETURN_IF_ERROR(writer.init()); + RETURN_IF_ERROR(writer.add_values("p", slices.data(), slices.size())); + RETURN_IF_ERROR(writer.finish()); + } + RETURN_IF_ERROR(index_writer.begin_close()); + return index_writer.finish_close(); + } + + void observe_flights(GramFlightGate* gate) { + for (const auto& reader : _readers) { + reader->set_single_flight_leader_before_compute_observer_for_test( + &GramFlightGate::leader, gate); + reader->set_single_flight_follower_joined_observer_for_test(&GramFlightGate::follower, + gate); + } + } + + std::vector run_queries(const std::vector& requests, + GramFlightGate* blocked_gate = nullptr, + size_t leaders = 0, size_t followers = 0) { + std::vector results(requests.size()); + std::barrier start(static_cast(requests.size() + 1)); + std::vector workers; + for (size_t i = 0; i < requests.size(); ++i) { + workers.emplace_back([&, i] { + SCOPED_ATTACH_TASK(_worker_tracker); + const auto& request = requests[i]; + RuntimeState runtime_state; + TQueryOptions options; + options.__set_query_type(TQueryType::SELECT); + options.__set_enable_inverted_index_query_cache(request.query_cache); + options.__set_enable_inverted_index_searcher_cache(request.searcher_cache); + runtime_state.set_query_options(options); + io::IOContext io_context; + auto context = std::make_shared(); + context->runtime_state = &runtime_state; + context->stats = &results[i].stats; + context->io_ctx = &io_context; + const auto value = Field::create_field(request.pattern); + start.arrive_and_wait(); + results[i].status = + request.reader->query(context, "p", value, request.type, results[i].bitmap); + }); + } + start.arrive_and_wait(); + if (blocked_gate != nullptr) { + EXPECT_TRUE(blocked_gate->wait_for_consumers(leaders, followers)); + // Always release before checking worker results, including timeout failures. + blocked_gate->release(); + } + for (auto& worker : workers) { + worker.join(); + } + return results; + } + + void expect_candidates(const GramCacheResult& result, const std::vector& expected) { + ASSERT_TRUE(result.status.ok()) << result.status; + ASSERT_NE(result.bitmap, nullptr); + EXPECT_EQ(std::vector(result.bitmap->begin(), result.bitmap->end()), expected); + } + + // Row 1 has every dense3 gram in "abcdef", but no dense4 gram. It is a + // permitted false positive for dense3 and distinguishes the physical schemes. + const std::vector _values {"abcdef", "abc-bcd-cde-def", "%abcdef%", + "unrelated", "xyzuvw", "abc"}; + IndexPolicyMgr _policy_mgr; + IndexPolicyMgr* _previous_policy_mgr = nullptr; + InvertedIndexQueryCache* _previous_query_cache = nullptr; + InvertedIndexSearcherCache* _previous_searcher_cache = nullptr; + std::unique_ptr _query_cache; + std::unique_ptr _searcher_cache; + std::shared_ptr _worker_tracker; + std::array _indexes; + std::shared_ptr _file_reader; + std::array, 2> _readers; + std::atomic _searcher_opens {0}; +}; + +TEST_F(SniiGramCacheTest, ColdIdenticalPatternsShareOneBitmapComputation) { + GramFlightGate gate; + observe_flights(&gate); + const std::vector requests(kConsumers, {_readers[0], "%abcdef%"}); + const auto results = run_queries(requests, &gate, 1, kConsumers - 1); + + EXPECT_EQ(gate.leaders(), 1); + EXPECT_EQ(gate.followers(), kConsumers - 1); + size_t searcher_misses = 0; + size_t query_inserts = 0; + for (const auto& result : results) { + expect_candidates(result, {0, 1, 2}); + EXPECT_EQ(result.bitmap, results.front().bitmap); + EXPECT_EQ(result.stats.inverted_index_query_cache_lookup, 1); + EXPECT_EQ(result.stats.inverted_index_query_cache_miss, 1); + EXPECT_EQ(result.stats.inverted_index_query_cache_hit, 0); + EXPECT_EQ(result.stats.inverted_index_searcher_cache_hit + + result.stats.inverted_index_searcher_cache_miss, + 1); + searcher_misses += result.stats.inverted_index_searcher_cache_miss; + query_inserts += result.stats.inverted_index_query_cache_insert; + } + EXPECT_EQ(query_inserts, 1); + EXPECT_GE(searcher_misses, 1); + // Logical readers are opened before joining the bitmap flight; concurrent + // cold searcher misses may therefore open more than one logical reader. + EXPECT_EQ(_searcher_opens.load(), searcher_misses); +} + +TEST_F(SniiGramCacheTest, WarmQueryHitsAvoidOpeningOrJoiningAndNewPatternsReuseSearcher) { + GramFlightGate cold_gate; + observe_flights(&cold_gate); + const std::vector requests(kConsumers, {_readers[0], "%abcdef%"}); + const auto cold = run_queries(requests, &cold_gate, 1, kConsumers - 1); + for (const auto& result : cold) { + expect_candidates(result, {0, 1, 2}); + } + const size_t cold_opens = _searcher_opens.load(); + + const auto warm = run_queries(requests); + for (const auto& result : warm) { + expect_candidates(result, {0, 1, 2}); + EXPECT_EQ(result.bitmap, cold.front().bitmap); + EXPECT_EQ(result.stats.inverted_index_query_cache_hit, 1); + EXPECT_EQ(result.stats.inverted_index_query_cache_miss, 0); + EXPECT_EQ(result.stats.inverted_index_query_cache_insert, 0); + EXPECT_EQ(result.stats.inverted_index_searcher_cache_hit, 0); + EXPECT_EQ(result.stats.inverted_index_searcher_cache_miss, 0); + } + EXPECT_EQ(cold_gate.leaders(), 1); + EXPECT_EQ(cold_gate.followers(), kConsumers - 1); + EXPECT_EQ(_searcher_opens.load(), cold_opens); + + GramFlightGate new_pattern_gate; + observe_flights(&new_pattern_gate); + const std::vector new_requests(kConsumers, {_readers[0], "%xyzuvw%"}); + const auto searcher_hits = run_queries(new_requests, &new_pattern_gate, 1, kConsumers - 1); + for (const auto& result : searcher_hits) { + expect_candidates(result, {4}); + EXPECT_EQ(result.stats.inverted_index_query_cache_hit, 0); + EXPECT_EQ(result.stats.inverted_index_query_cache_miss, 1); + EXPECT_EQ(result.stats.inverted_index_searcher_cache_hit, 1); + EXPECT_EQ(result.stats.inverted_index_searcher_cache_miss, 0); + } + EXPECT_EQ(new_pattern_gate.leaders(), 1); + EXPECT_EQ(new_pattern_gate.followers(), kConsumers - 1); + EXPECT_EQ(_searcher_opens.load(), cold_opens); +} + +TEST_F(SniiGramCacheTest, PhysicalSchemesInOneContainerKeepFlightsAndBothCachesSeparate) { + for (size_t i = 0; i < _indexes.size(); ++i) { + auto opened = _file_reader->open_snii_index(&_indexes[i]); + ASSERT_TRUE(opened.has_value()) << opened.error(); + const auto& scheme = opened.value()->gram_scheme(); + ASSERT_TRUE(scheme.has_value()); + ASSERT_EQ(scheme->min_len, 3 + i); + } + ASSERT_NE(_file_reader->get_index_file_cache_key(&_indexes[0]), + _file_reader->get_index_file_cache_key(&_indexes[1])); + + std::vector requests; + for (size_t i = 0; i < kConsumers; ++i) { + requests.push_back({_readers[i % 2], "%abcdef%"}); + } + GramFlightGate cold_gate; + observe_flights(&cold_gate); + const auto cold = run_queries(requests, &cold_gate, 2, kConsumers - 2); + size_t inserts = 0; + for (size_t i = 0; i < cold.size(); ++i) { + expect_candidates(cold[i], i % 2 == 0 ? std::vector {0, 1, 2} + : std::vector {0, 2}); + EXPECT_EQ(cold[i].bitmap, cold[i % 2].bitmap); + EXPECT_EQ(cold[i].stats.inverted_index_query_cache_miss, 1); + inserts += cold[i].stats.inverted_index_query_cache_insert; + } + EXPECT_NE(cold[0].bitmap, cold[1].bitmap); + EXPECT_EQ(inserts, 2); + EXPECT_EQ(cold_gate.leaders(), 2); + EXPECT_EQ(cold_gate.followers(), kConsumers - 2); + + const size_t cold_opens = _searcher_opens.load(); + const auto warm = run_queries(requests); + for (size_t i = 0; i < warm.size(); ++i) { + expect_candidates(warm[i], i % 2 == 0 ? std::vector {0, 1, 2} + : std::vector {0, 2}); + EXPECT_EQ(warm[i].bitmap, cold[i % 2].bitmap); + EXPECT_EQ(warm[i].stats.inverted_index_query_cache_hit, 1); + EXPECT_EQ(warm[i].stats.inverted_index_searcher_cache_hit, 0); + EXPECT_EQ(warm[i].stats.inverted_index_searcher_cache_miss, 0); + } + EXPECT_EQ(_searcher_opens.load(), cold_opens); + EXPECT_EQ(cold_gate.leaders(), 2); + EXPECT_EQ(cold_gate.followers(), kConsumers - 2); + + // A different raw pattern misses the result cache and must retrieve the + // correct physical scheme from the already warm searcher cache. + for (auto& request : requests) { + request.pattern = "%%abcdef%%"; + } + GramFlightGate searcher_gate; + observe_flights(&searcher_gate); + const auto searcher_hits = run_queries(requests, &searcher_gate, 2, kConsumers - 2); + for (size_t i = 0; i < searcher_hits.size(); ++i) { + expect_candidates(searcher_hits[i], i % 2 == 0 ? std::vector {0, 1, 2} + : std::vector {0, 2}); + EXPECT_EQ(searcher_hits[i].stats.inverted_index_query_cache_miss, 1); + EXPECT_EQ(searcher_hits[i].stats.inverted_index_searcher_cache_hit, 1); + EXPECT_EQ(searcher_hits[i].stats.inverted_index_searcher_cache_miss, 0); + } + EXPECT_EQ(searcher_gate.leaders(), 2); + EXPECT_EQ(searcher_gate.followers(), kConsumers - 2); + EXPECT_EQ(_searcher_opens.load(), cold_opens); +} + +TEST_F(SniiGramCacheTest, IdenticalRawLikeAndRegexpPatternsKeepFlightsAndResultsSeparate) { + std::vector requests; + for (size_t i = 0; i < kConsumers; ++i) { + requests.push_back({_readers[0], "%abcdef%", + i % 2 == 0 ? InvertedIndexQueryType::LIKE_GRAM_QUERY + : InvertedIndexQueryType::REGEXP_GRAM_QUERY}); + } + GramFlightGate gate; + observe_flights(&gate); + const auto cold = run_queries(requests, &gate, 2, kConsumers - 2); + size_t inserts = 0; + for (size_t i = 0; i < cold.size(); ++i) { + // '%' is a wildcard in LIKE but a literal byte in REGEXP. Row 2 is + // the only candidate containing both the leading and trailing '%'. + expect_candidates(cold[i], + i % 2 == 0 ? std::vector {0, 1, 2} : std::vector {2}); + EXPECT_EQ(cold[i].bitmap, cold[i % 2].bitmap); + EXPECT_EQ(cold[i].stats.inverted_index_query_cache_miss, 1); + inserts += cold[i].stats.inverted_index_query_cache_insert; + } + EXPECT_NE(cold[0].bitmap, cold[1].bitmap); + EXPECT_EQ(inserts, 2); + EXPECT_EQ(gate.leaders(), 2); + EXPECT_EQ(gate.followers(), kConsumers - 2); + + const size_t cold_opens = _searcher_opens.load(); + const auto warm = run_queries(requests); + for (size_t i = 0; i < warm.size(); ++i) { + expect_candidates(warm[i], + i % 2 == 0 ? std::vector {0, 1, 2} : std::vector {2}); + EXPECT_EQ(warm[i].bitmap, cold[i % 2].bitmap); + EXPECT_EQ(warm[i].stats.inverted_index_query_cache_hit, 1); + EXPECT_EQ(warm[i].stats.inverted_index_query_cache_miss, 0); + EXPECT_EQ(warm[i].stats.inverted_index_searcher_cache_hit, 0); + EXPECT_EQ(warm[i].stats.inverted_index_searcher_cache_miss, 0); + } + EXPECT_EQ(_searcher_opens.load(), cold_opens); + EXPECT_EQ(gate.leaders(), 2); + EXPECT_EQ(gate.followers(), kConsumers - 2); +} + +TEST_F(SniiGramCacheTest, UnprunablePatternsReleaseFollowersWithoutCachingResults) { + for (const auto& [pattern, type] : std::vector> { + {"%", InvertedIndexQueryType::LIKE_GRAM_QUERY}, + {".*", InvertedIndexQueryType::REGEXP_GRAM_QUERY}, + {"[", InvertedIndexQueryType::REGEXP_GRAM_QUERY}}) { + SCOPED_TRACE(pattern); + const std::vector requests(kConsumers, {_readers[0], pattern, type}); + for (size_t attempt = 0; attempt < 2; ++attempt) { + SCOPED_TRACE(attempt); + GramFlightGate gate; + observe_flights(&gate); + const auto results = run_queries(requests, &gate, 1, kConsumers - 1); + // These observers prove the consumers really waited on a leader + // that returned non-OK. Followers then retry independently, and a + // second cohort must be able to lead the same physical query again. + EXPECT_EQ(gate.leaders(), 1); + EXPECT_EQ(gate.followers(), kConsumers - 1); + for (const auto& result : results) { + EXPECT_TRUE(result.status.is()) + << result.status; + EXPECT_EQ(result.bitmap, nullptr); + EXPECT_EQ(result.stats.inverted_index_query_cache_hit, 0); + EXPECT_EQ(result.stats.inverted_index_query_cache_miss, 1); + EXPECT_EQ(result.stats.inverted_index_query_cache_insert, 0); + } + } + } +} + +class SniiGramCacheOptionsTest : public SniiGramCacheTest, + public testing::WithParamInterface> {}; + +TEST_P(SniiGramCacheOptionsTest, RepeatedPatternsRespectBothCacheSwitches) { + const auto [query_cache, searcher_cache] = GetParam(); + GramFlightGate gate; + gate.release(); + observe_flights(&gate); + const std::vector requests {{_readers[0], "%abcdef%", + InvertedIndexQueryType::LIKE_GRAM_QUERY, + query_cache, searcher_cache}}; + const auto cold = run_queries(requests); + expect_candidates(cold.front(), {0, 1, 2}); + EXPECT_EQ(cold.front().stats.inverted_index_query_cache_hit, 0); + EXPECT_EQ(cold.front().stats.inverted_index_query_cache_miss, query_cache ? 1 : 0); + EXPECT_EQ(cold.front().stats.inverted_index_query_cache_insert, query_cache ? 1 : 0); + EXPECT_EQ(cold.front().stats.inverted_index_searcher_cache_miss, 1); + EXPECT_EQ(_searcher_opens.load(), 1); + + const auto repeated = run_queries(requests); + expect_candidates(repeated.front(), {0, 1, 2}); + const auto& stats = repeated.front().stats; + EXPECT_EQ(stats.inverted_index_query_cache_lookup, query_cache ? 1 : 0); + EXPECT_EQ(stats.inverted_index_query_cache_hit, query_cache ? 1 : 0); + EXPECT_EQ(stats.inverted_index_query_cache_miss, 0); + EXPECT_EQ(stats.inverted_index_query_cache_insert, 0); + EXPECT_EQ(stats.inverted_index_searcher_cache_hit, !query_cache && searcher_cache ? 1 : 0); + EXPECT_EQ(stats.inverted_index_searcher_cache_miss, !query_cache && !searcher_cache ? 1 : 0); + EXPECT_EQ(_searcher_opens.load(), !query_cache && !searcher_cache ? 2 : 1); + // Query-cache disablement still allows independent calls to lead flights. + EXPECT_EQ(gate.leaders(), query_cache ? 1 : 2); + EXPECT_EQ(gate.followers(), 0); +} + +INSTANTIATE_TEST_SUITE_P(CacheSwitches, SniiGramCacheOptionsTest, + testing::Values(std::pair {false, false}, std::pair {false, true}, + std::pair {true, false}, std::pair {true, true})); + +} // namespace +} // namespace doris::segment_v2 diff --git a/be/test/storage/segment/segment_iterator_conjunct_short_circuit_test.cpp b/be/test/storage/segment/segment_iterator_conjunct_short_circuit_test.cpp index 43ce939cf11156..f3d74ca6ecc82e 100644 --- a/be/test/storage/segment/segment_iterator_conjunct_short_circuit_test.cpp +++ b/be/test/storage/segment/segment_iterator_conjunct_short_circuit_test.cpp @@ -60,7 +60,16 @@ namespace { // result bitmap, mimicking how real match exprs publish results. class BitmapEvalExpr : public VExpr { public: - explicit BitmapEvalExpr(std::vector rows) : _rows(std::move(rows)) { + // kExact is a regular inverted-index result, which the iterator consumes: the expression is + // erased from the push-down list. kApproximate is a gram push-down -- a superset of + // candidate rows that only prunes _row_bitmap while the expression stays pushed down. + // kApproximateNoBitmap is the default-constructed "no result" shape, which the approximate + // map is not supposed to hold but which must not crash the reader either. kNone publishes + // nothing, as an index that could not be used does. + enum class ResultKind { kNone, kExact, kApproximate, kApproximateNoBitmap }; + + explicit BitmapEvalExpr(std::vector rows, ResultKind kind = ResultKind::kExact) + : _rows(std::move(rows)), _kind(kind) { set_node_type(TExprNodeType::MATCH_PRED); _data_type = std::make_shared(); } @@ -79,12 +88,27 @@ class BitmapEvalExpr : public VExpr { Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override { ++_eval_count; + auto index_context = context->get_index_context(); + if (_kind == ResultKind::kNone) { + return Status::OK(); + } + if (_kind == ResultKind::kApproximateNoBitmap) { + InvertedIndexResultBitmap empty; + empty.set_approximate(true); + index_context->set_approx_index_result_for_expr(this, std::move(empty)); + return Status::OK(); + } auto data = std::make_shared(); for (uint32_t row : _rows) { data->add(row); } InvertedIndexResultBitmap result(std::move(data), std::make_shared()); - context->get_index_context()->set_index_result_for_expr(this, std::move(result)); + if (_kind == ResultKind::kApproximate) { + result.set_approximate(true); + index_context->set_approx_index_result_for_expr(this, std::move(result)); + } else { + index_context->set_index_result_for_expr(this, std::move(result)); + } return Status::OK(); } @@ -104,6 +128,7 @@ class BitmapEvalExpr : public VExpr { private: std::vector _rows; + ResultKind _kind; int _eval_count = 0; int _ann_eval_count = 0; }; @@ -325,4 +350,123 @@ TEST_F(SegmentIteratorConjunctShortCircuitTest, progressive_intersection_keeps_g EXPECT_EQ(_stats.inverted_index_conjuncts_short_circuited, 0); } +// --------------------------------------------------------------------------------------------- +// _apply_approx_index_result: the single door through which an approximate (superset) index +// result reaches _row_bitmap. +// --------------------------------------------------------------------------------------------- + +// An approximate result narrows the candidate rows and does nothing else: the expression stays +// in _common_expr_ctxs_push_down, because rows inside the bitmap have not been proved to match +// and _execute_common_expr still has to re-verify every one of them. The two gram profile +// counters account for what survived and what was pruned. +TEST_F(SegmentIteratorConjunctShortCircuitTest, approx_result_narrows_bitmap_but_keeps_the_expr) { + _iter->_row_bitmap.addRange(0, 10); + + auto approx_expr = std::make_shared(std::vector {1, 3, 5, 42}, + BitmapEvalExpr::ResultKind::kApproximate); + _iter->_common_expr_ctxs_push_down = {make_bitmap_ctx(approx_expr)}; + + ASSERT_TRUE(_iter->_apply_index_expr().ok()); + + EXPECT_EQ(_iter->_row_bitmap.cardinality(), 3); + EXPECT_TRUE(_iter->_row_bitmap.contains(1)); + EXPECT_TRUE(_iter->_row_bitmap.contains(3)); + EXPECT_TRUE(_iter->_row_bitmap.contains(5)); + EXPECT_EQ(_iter->_common_expr_ctxs_push_down.size(), 1) + << "an approximate result must never consume the expression"; + EXPECT_EQ(_stats.gram_index_candidate_rows, 3); + EXPECT_EQ(_stats.rows_gram_index_filtered, 7); +} + +// An approximate result that prunes nothing still counts its candidates and reports zero rows +// filtered, rather than a negative number from the unsigned difference. +TEST_F(SegmentIteratorConjunctShortCircuitTest, approx_result_that_prunes_nothing_filters_none) { + _iter->_row_bitmap.addRange(0, 3); + + auto approx_expr = std::make_shared(std::vector {0, 1, 2, 3}, + BitmapEvalExpr::ResultKind::kApproximate); + _iter->_common_expr_ctxs_push_down = {make_bitmap_ctx(approx_expr)}; + + ASSERT_TRUE(_iter->_apply_index_expr().ok()); + + EXPECT_EQ(_iter->_row_bitmap.cardinality(), 3); + EXPECT_EQ(_stats.gram_index_candidate_rows, 3); + EXPECT_EQ(_stats.rows_gram_index_filtered, 0); +} + +// Nothing was published for this expression: the index could not be used, which may only cost +// the acceleration. Every row survives and neither counter moves. +TEST_F(SegmentIteratorConjunctShortCircuitTest, no_published_result_leaves_the_bitmap_untouched) { + _iter->_row_bitmap.addRange(0, 4); + + auto silent_expr = std::make_shared(std::vector {1}, + BitmapEvalExpr::ResultKind::kNone); + _iter->_common_expr_ctxs_push_down = {make_bitmap_ctx(silent_expr)}; + + ASSERT_TRUE(_iter->_apply_index_expr().ok()); + + EXPECT_EQ(_iter->_row_bitmap.cardinality(), 4); + EXPECT_EQ(_iter->_common_expr_ctxs_push_down.size(), 1); + EXPECT_EQ(_stats.gram_index_candidate_rows, 0); + EXPECT_EQ(_stats.rows_gram_index_filtered, 0); +} + +// A default-constructed InvertedIndexResultBitmap carries a null data bitmap. It is not a shape +// the approximate map is meant to hold, but dereferencing it would be a segfault, so the null +// check has to turn it into "no pruning" instead. +TEST_F(SegmentIteratorConjunctShortCircuitTest, approx_result_without_a_bitmap_prunes_nothing) { + _iter->_row_bitmap.addRange(0, 4); + + auto approx_expr = std::make_shared( + std::vector {1}, BitmapEvalExpr::ResultKind::kApproximateNoBitmap); + _iter->_common_expr_ctxs_push_down = {make_bitmap_ctx(approx_expr)}; + + ASSERT_TRUE(_iter->_apply_index_expr().ok()); + + EXPECT_EQ(_iter->_row_bitmap.cardinality(), 4); + EXPECT_EQ(_iter->_common_expr_ctxs_push_down.size(), 1); + EXPECT_EQ(_stats.gram_index_candidate_rows, 0); + EXPECT_EQ(_stats.rows_gram_index_filtered, 0); +} + +// The contrast that pins the two paths apart: an exact result is consumed -- the expression +// leaves the push-down list because the index decided every row -- and it must not show up on +// the gram counters, which describe approximate pruning only. +TEST_F(SegmentIteratorConjunctShortCircuitTest, exact_result_consumes_and_skips_gram_counters) { + _iter->_row_bitmap.addRange(0, 5); + + auto exact_expr = std::make_shared(std::vector {1, 3}); + _iter->_common_expr_ctxs_push_down = {make_bitmap_ctx(exact_expr)}; + + ASSERT_TRUE(_iter->_apply_index_expr().ok()); + + EXPECT_EQ(_iter->_row_bitmap.cardinality(), 2); + EXPECT_TRUE(_iter->_common_expr_ctxs_push_down.empty()) + << "an exact result must consume the expression"; + EXPECT_EQ(_stats.gram_index_candidate_rows, 0); + EXPECT_EQ(_stats.rows_gram_index_filtered, 0); +} + +// An approximate conjunct that empties the candidate set short-circuits exactly like an exact +// one: zero rows satisfy the whole conjunction, so the remaining conjuncts are never evaluated +// and the list is cleared, while the counters still describe what the approximate result did. +TEST_F(SegmentIteratorConjunctShortCircuitTest, approx_result_emptying_the_bitmap_short_circuits) { + _iter->_row_bitmap.addRange(0, 3); + + auto approx_expr = std::make_shared(std::vector {7, 8}, + BitmapEvalExpr::ResultKind::kApproximate); + auto expensive_expr = std::make_shared(std::vector {0, 1}); + _iter->_common_expr_ctxs_push_down = {make_bitmap_ctx(approx_expr), + make_bitmap_ctx(expensive_expr)}; + + ASSERT_TRUE(_iter->_apply_index_expr().ok()); + + EXPECT_TRUE(_iter->_row_bitmap.isEmpty()); + EXPECT_EQ(expensive_expr->eval_count(), 0); + EXPECT_TRUE(_iter->_common_expr_ctxs_push_down.empty()); + EXPECT_TRUE(_iter->_index_conjuncts_proved_empty); + EXPECT_EQ(_stats.gram_index_candidate_rows, 0); + EXPECT_EQ(_stats.rows_gram_index_filtered, 3); +} + } // namespace doris::segment_v2 From 2223c91c550c997ee6b76077c5112ed524d1cf7b Mon Sep 17 00:00:00 2001 From: airborne12 Date: Thu, 10 Sep 2026 20:55:19 +0800 Subject: [PATCH 05/36] [feature](fe) validate gram tokenizer properties, gram-family analyzers and gram index defaults - NGramTokenizerValidator accepts mode (auto|sparse|dense), density in [0.001, 1] and lower_case, with the gram-family ranges of min_gram / max_gram; without mode the legacy behaviour is untouched. - IndexPolicyMgr rejects a gram tokenizer combined with any char filter or token filter, mirroring BE (use the tokenizer's own lower_case instead of a lowercase filter, because folding must precede the gram boundaries), and resolves an analyzer's gram mode for the index checks. - InvertedIndexUtil requires inverted_index_storage_format = SNII for a gram-family index, rejects support_phrase = true and index-level char filters, and defaults support_phrase to false, also for CREATE INDEX and ALTER TABLE ADD INDEX. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01METuP3aVRn8dfCU62PivnF --- .../doris/alter/SchemaChangeHandler.java | 6 + .../doris/analysis/InvertedIndexUtil.java | 68 +++- .../doris/indexpolicy/IndexPolicyMgr.java | 138 ++++++++ .../indexpolicy/NGramTokenizerValidator.java | 120 ++++++- .../plans/commands/info/IndexDefinition.java | 28 ++ .../indexpolicy/GramDdlValidationTest.java | 331 ++++++++++++++++++ .../indexpolicy/PolicyValidatorTests.java | 212 +++++++++++ 7 files changed, 899 insertions(+), 4 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/indexpolicy/GramDdlValidationTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 4aed9785b5c2aa..de685e3cfab7d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -3314,6 +3314,12 @@ private boolean processAddIndex(CreateIndexOp createIndexOp, OlapTable olapTable } } + // CreateIndexOp#validate materializes the IndexDefinition into alterIndex before + // checkColumn runs, so the property defaults that are only filled in during checkColumn + // (currently the gram family's support_phrase=false) have to be written back here, or the + // index that is persisted and shipped to BE would lose them. + indexDef.applyPropertiesTo(alterIndex); + // the column name in CreateIndexClause is not check case sensitivity, // when send index description to BE, there maybe cannot find column by name, // so here update column name in CreateIndexClause after checkColumn for indexDef, diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java index 38b10ca17f9168..efaa7425ed525a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java @@ -41,6 +41,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; public class InvertedIndexUtil { @@ -267,7 +268,8 @@ private static void checkInvertedIndexProperties(Map properties, + "or 'normalizer' for text normalization without tokenization."); } - checkAnalyzerName(analyzerName, colType); + checkAnalyzerName(analyzerName, colType, invertedIndexFileStorageFormat, supportPhrase); + applyGramFamilyIndexDefaults(analyzerName, properties); checkNormalizerName(normalizerName, colType); if (parser != null @@ -395,7 +397,9 @@ private static String resolveAnalyzerName(String name, Set builtins) { return policy == null ? trimmedName.toLowerCase(Locale.ROOT) : policy.getName(); } - private static void checkAnalyzerName(String analyzerName, PrimitiveType colType) throws AnalysisException { + private static void checkAnalyzerName(String analyzerName, PrimitiveType colType, + TInvertedIndexFileStorageFormat storageFormat, String supportPhrase) + throws AnalysisException { if (analyzerName == null || analyzerName.isEmpty()) { return; } @@ -404,12 +408,70 @@ private static void checkAnalyzerName(String analyzerName, PrimitiveType colType + " is not supported for column of type " + colType); } try { - Env.getCurrentEnv().getIndexPolicyMgr().validateAnalyzerExists(analyzerName); + IndexPolicyMgr indexPolicyMgr = Env.getCurrentEnv().getIndexPolicyMgr(); + indexPolicyMgr.validateAnalyzerExists(analyzerName); + // Gram-family analyzer (an ngram tokenizer carrying mode, see + // IndexPolicyMgr#resolveGramTokenizerMode): BE builds sparse/dense gram postings for it + // only on SNII, and those postings carry no positions, so phrase queries are impossible. + Optional gramMode = indexPolicyMgr.resolveGramTokenizerMode(analyzerName); + if (gramMode.isPresent()) { + if (colType.isArrayType()) { + throw new AnalysisException("gram tokenizer (mode=" + gramMode.get() + + ") analyzer '" + analyzerName + "' does not support ARRAY columns"); + } + if (!colType.isCharFamily()) { + throw new AnalysisException("gram tokenizer (mode=" + gramMode.get() + + ") analyzer '" + analyzerName + + "' is supported only on scalar CHAR, VARCHAR, or STRING columns"); + } + if (storageFormat != TInvertedIndexFileStorageFormat.SNII) { + throw new AnalysisException("gram tokenizer (mode=" + gramMode.get() + + ") requires inverted_index_storage_format = SNII"); + } + if ("true".equals(supportPhrase)) { + throw new AnalysisException( + "gram tokenizer index does not support phrase (support_phrase must be false)"); + } + } } catch (DdlException e) { throw new AnalysisException("Invalid custom analyzer: " + e.getMessage()); } } + /** + * Constraints a gram-family analyzer (an ngram tokenizer carrying mode) enforces at the index + * property level: + * 1) an index-level char_filter (char_filter_type/pattern/replacement) conflicts with the + * semantics of gram split boundaries (the character replacement happens after the tokenizer + * has already split by the gram rule, which breaks reproducibility), so it is rejected; + * 2) when support_phrase is not given explicitly it defaults to "false", overriding the general + * rule of the {@link Index} constructor that "an analyzer implies true" -- a gram index is + * forced to docs-only on the BE side and has no positions for a phrase query to use. + * + *

The {@code properties} held by the caller ({@link #checkInvertedIndexProperties}) and the + * {@link IndexDefinition} field are the same mutable Map reference, so the defaults written + * here are seen when {@code IndexDefinition#translateToCatalogStyle} builds the {@link Index}. + */ + private static void applyGramFamilyIndexDefaults(String analyzerName, Map properties) + throws AnalysisException { + if (analyzerName == null || analyzerName.isEmpty()) { + return; + } + Optional gramMode = Env.getCurrentEnv().getIndexPolicyMgr().resolveGramTokenizerMode(analyzerName); + if (!gramMode.isPresent()) { + return; + } + if (properties.get(INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE) != null + || properties.get(INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN) != null + || properties.get(INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT) != null) { + throw new AnalysisException("char_filter cannot be used with gram tokenizer (mode=" + + gramMode.get() + ")"); + } + if (properties.get(INVERTED_INDEX_SUPPORT_PHRASE_KEY) == null) { + properties.put(INVERTED_INDEX_SUPPORT_PHRASE_KEY, "false"); + } + } + private static void checkNormalizerName(String normalizerName, PrimitiveType colType) throws AnalysisException { if (normalizerName == null || normalizerName.isEmpty()) { return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java index bccd7e3f215f31..71f7e31482ba50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java @@ -43,6 +43,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @@ -269,6 +270,52 @@ private void validateReferencedFilterTypesLocked(String filterNames, IndexPolicy } } + /** + * Decides whether {@code analyzerName} is a "gram family" analyzer: an ngram tokenizer that + * explicitly carries mode (sparse/auto/dense). On the BE side a gram-family analyzer produces + * only sparse/dense gram postings, which only the SNII storage format can read and which carry + * no positions (no phrase queries); the caller (currently + * {@code InvertedIndexUtil.checkAnalyzerName}) uses this to constrain the index properties. + * + *

Returns {@code Optional.empty()} when this is not a custom analyzer (null, a built-in + * analyzer name, not found, wrong type), when the tokenizer is not ngram, or when the ngram + * carries no mode (the legacy min_gram/max_gram usage). On a hit it returns the lower-cased mode + * value. + */ + public Optional resolveGramTokenizerMode(String analyzerName) { + if (analyzerName == null) { + return Optional.empty(); + } + String normalizedName = normalizeKey(analyzerName); + if (IndexPolicy.BUILTIN_ANALYZERS.contains(normalizedName)) { + return Optional.empty(); + } + + readLock(); + try { + IndexPolicy analyzer = nameToIndexPolicy.get(normalizedName); + if (analyzer == null || analyzer.getType() != IndexPolicyTypeEnum.ANALYZER) { + return Optional.empty(); + } + String tokenizerRef = analyzer.getProperties().get(IndexPolicy.PROP_TOKENIZER); + ResolvedAnalyzerComponent tokenizer; + try { + tokenizer = resolveAnalyzerComponentLocked(tokenizerRef, IndexPolicyTypeEnum.TOKENIZER); + } catch (DdlException e) { + // A dangling tokenizer reference (dropped, say) is not this method's problem to + // report; other validation paths handle it. + return Optional.empty(); + } + if (!"ngram".equals(tokenizer.componentType)) { + return Optional.empty(); + } + String mode = tokenizer.properties.get("mode"); + return (mode == null || mode.isEmpty()) ? Optional.empty() : Optional.of(mode.toLowerCase()); + } finally { + readUnlock(); + } + } + public void validateNormalizerExists(String normalizerName) throws DdlException { readLock(); try { @@ -426,6 +473,97 @@ private void validateAnalyzerProperties(Map properties) throws D validatePolicyReference(filter, IndexPolicyTypeEnum.CHAR_FILTER); } } + validateGramFamilyAnalyzerLocked(properties); + } + + // Gram-family constraint: when the tokenizer is ngram and carries mode (sparse/auto/dense + // -- any value, dense included), the analyzer may not stack any char filter or token + // filter on top -- the gram split boundaries must be drawn entirely by the tokenizer itself + // (case folding and the like go through the tokenizer's own lower_case property and happen + // before the split), otherwise a stacked filter would transform the text or the terms + // around the gram boundaries and break the reproducible semantics of a sparse/dense gram + // index. + // + // This mirrors BE exactly. CustomAnalyzerProvider only derives a gram scheme when + // `char_filter_configs` and `token_filter_configs` are both empty, so an analyzer that FE + // let through with a char filter would be constrained here like a gram index (SNII + // required, support_phrase forced to false) while BE built one that can never accelerate + // LIKE/REGEXP -- silently degraded, with no error anywhere. + private void validateGramFamilyAnalyzerLocked(Map properties) throws DdlException { + ResolvedAnalyzerComponent tokenizer = resolveAnalyzerComponentLocked( + properties.get(IndexPolicy.PROP_TOKENIZER), IndexPolicyTypeEnum.TOKENIZER); + if (!"ngram".equals(tokenizer.componentType)) { + return; + } + String gramMode = tokenizer.properties.get("mode"); + if (gramMode == null || gramMode.isEmpty()) { + return; + } + String charFilterNames = properties.get(IndexPolicy.PROP_CHAR_FILTER); + if (charFilterNames != null && !charFilterNames.isEmpty()) { + throw new DdlException("char filter '" + charFilterNames + + "' cannot be combined with ngram tokenizer mode=" + gramMode.toLowerCase() + + " (gram-family analyzers must be tokenizer-only)"); + } + String tokenFilterNames = properties.get(IndexPolicy.PROP_TOKEN_FILTER); + if (tokenFilterNames == null || tokenFilterNames.isEmpty()) { + return; + } + // Any token filter in the chain is rejected; if one of them is lowercase, prefer that + // more specific hint (even when it is not the first), because "use the tokenizer's own + // lower_case" is the only correct fix for that spelling. + String[] tokenFilters = tokenFilterNames.split(",\\s*"); + for (String tokenFilterName : tokenFilters) { + ResolvedAnalyzerComponent tokenFilter = resolveAnalyzerComponentLocked( + tokenFilterName, IndexPolicyTypeEnum.TOKEN_FILTER); + if ("lowercase".equals(tokenFilter.componentType)) { + throw new DdlException("lowercase token filter cannot be combined with ngram tokenizer mode=" + + gramMode.toLowerCase() + + "; use the tokenizer's own lower_case=true " + + "(folding must happen before gram boundaries)"); + } + } + throw new DdlException("token filter '" + tokenFilters[0] + + "' cannot be combined with ngram tokenizer mode=" + gramMode.toLowerCase() + + " (gram-family analyzers must be tokenizer-only)"); + } + + private ResolvedAnalyzerComponent resolveAnalyzerComponentLocked(String referenceName, + IndexPolicyTypeEnum expectedType) throws DdlException { + String normalizedName = normalizeKey(referenceName); + boolean builtin = expectedType == IndexPolicyTypeEnum.TOKENIZER + ? IndexPolicy.BUILTIN_TOKENIZERS.contains(normalizedName) + : IndexPolicy.BUILTIN_TOKEN_FILTERS.contains(normalizedName); + if (builtin) { + return new ResolvedAnalyzerComponent(referenceName, normalizedName, + Map.of(IndexPolicy.PROP_TYPE, normalizedName)); + } + + IndexPolicy policy = nameToIndexPolicy.get(normalizedName); + if (policy == null) { + throw new DdlException("Referenced " + expectedType + " policy '" + + referenceName + "' does not exist"); + } + if (policy.getType() != expectedType) { + throw new DdlException("Referenced policy '" + referenceName + "' is of type " + + policy.getType() + " but expected " + expectedType); + } + return new ResolvedAnalyzerComponent(referenceName, + policy.getProperties().get(IndexPolicy.PROP_TYPE), + policy.getProperties()); + } + + private static final class ResolvedAnalyzerComponent { + private final String referenceName; + private final String componentType; + private final Map properties; + + private ResolvedAnalyzerComponent(String referenceName, String componentType, + Map properties) { + this.referenceName = referenceName; + this.componentType = componentType; + this.properties = properties; + } } private void validateNormalizerProperties(Map properties) throws DdlException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NGramTokenizerValidator.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NGramTokenizerValidator.java index df16c633e8add1..d7d8ea6dd1a05a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NGramTokenizerValidator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NGramTokenizerValidator.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; public class NGramTokenizerValidator extends BasePolicyValidator { // A configured range can emit one token per gram size at every input position. @@ -33,13 +34,39 @@ public class NGramTokenizerValidator extends BasePolicyValidator { static final int MAX_NGRAM_SIZE = 1024; private static final Set ALLOWED_PROPS = ImmutableSet.of( - "type", "min_gram", "max_gram", "max_ngram_diff", "token_chars", "custom_token_chars"); + "type", "min_gram", "max_gram", "max_ngram_diff", "token_chars", "custom_token_chars", + "mode", "density", "lower_case"); private static final Set VALID_TOKEN_CHARS = ImmutableSet.of( "letter", "digit", "whitespace", "punctuation", "symbol", "custom"); private final boolean enforceAbsoluteSizeLimit; + // The mode values supported by the gram family (sparse/dense gram indexes). + // They must match BE's `gram_scheme.cpp::GramScheme::from_properties` exactly: BE compares the + // strings literally (no trimming, no case folding), so FE also accepts only strictly lower-case + // literals without whitespace: " Sparse " / "SPARSE" are rejected at DDL time instead of letting + // the DDL pass and having BE report InvalidArgument at write time (FE does no implicit + // normalization, which would leave the persisted policy properties differing from what the user + // wrote). + private static final Set VALID_MODES = ImmutableSet.of("auto", "sparse", "dense"); + + // Value domains of the gram-family parameters, mirroring BE's `gram_scheme.cpp::from_properties`. + private static final int MIN_GRAM_LOWER_BOUND = 1; + private static final int MIN_GRAM_UPPER_BOUND = 64; + private static final int MAX_GRAM_LOWER_BOUND = 1; + private static final int MAX_GRAM_UPPER_BOUND = 256; + private static final double MIN_DENSITY = 0.001; + // Persist a decimal spelling that BE can parse verbatim: Java-only type suffixes or + // silently trimmed trailing whitespace must not pass DDL validation. + private static final Pattern DECIMAL_PATTERN = Pattern.compile( + "[+-]?(?:[0-9]+(?:\\.[0-9]*)?|\\.[0-9]+)(?:[eE][+-]?[0-9]+)?"); + // Same reason as DECIMAL_PATTERN, for the integer properties: Integer.parseInt resolves any + // Unicode decimal digit through Character.digit, while BE's parse_uint uses strtol and accepts + // ASCII only. Without this guard a full-width or Arabic-Indic spelling of a number would pass + // DDL validation and then fail at index build time with an opaque analyzer error. + private static final Pattern INTEGER_PATTERN = Pattern.compile("[+-]?[0-9]+"); + public NGramTokenizerValidator() { this(true); } @@ -68,6 +95,21 @@ protected String getTypeName() { @Override protected void validateSpecific(Map props) throws DdlException { + // The new gram-family parameters: once mode is given this is an auto/sparse/dense gram index, + // whose validation rules are entirely independent of legacy ngram, so validate and return + // without reusing any of the legacy rules below. + String mode = props.get("mode"); + if (mode != null) { + validateGramMode(props, mode); + return; + } + // Without mode, density/lower_case are meaningless, so reject them outright + for (String key : new String[] {"density", "lower_case"}) { + if (props.containsKey(key)) { + throw new DdlException("ngram tokenizer parameter '" + key + "' requires mode = auto|sparse|dense"); + } + } + int minGram = 1; if (props.containsKey("min_gram")) { try { @@ -156,4 +198,80 @@ protected void validateSpecific(Map props) throws DdlException { } } } + + /** + * Validates the gram-family (auto/sparse/dense) parameters: the value domain of mode itself, the + * defaults and the ordering of min/max_gram, the value domains of density/lower_case, and the + * mutual exclusion of the token_chars family with mode. + * An empty mode string (which the allow-list stage does not catch) is rejected here too, for not + * belonging to VALID_MODES. + * + *

Every value domain mirrors BE's `gram_scheme.cpp::GramScheme::from_properties` entry by entry: + * min_gram in [1, 64], max_gram in [1, 256], density in [0.001, 1]. + * FE rejects out-of-range values up front, so a DDL cannot pass only for BE to report + * InvalidArgument when it parses the gram scheme. + */ + private void validateGramMode(Map props, String mode) throws DdlException { + if (!VALID_MODES.contains(mode)) { + throw new DdlException("ngram tokenizer mode must be one of " + VALID_MODES + + ", got: '" + mode + "'" + (mode.isEmpty() ? " (empty)" : "")); + } + int minGram = parseIntInRange(props, "min_gram", 3, MIN_GRAM_LOWER_BOUND, MIN_GRAM_UPPER_BOUND); + int maxGram = parseIntInRange(props, "max_gram", 16, MAX_GRAM_LOWER_BOUND, MAX_GRAM_UPPER_BOUND); + if (minGram > maxGram) { + throw new DdlException("min_gram (" + minGram + ") must be <= max_gram (" + maxGram + ")"); + } + if (props.containsKey("density")) { + double density = parseDouble(props.get("density"), "density"); + if (!(density >= MIN_DENSITY && density <= 1.0)) { + throw new DdlException("density must be in [0.001, 1], got: " + props.get("density")); + } + } + if (props.containsKey("lower_case") && !props.get("lower_case").matches("true|false")) { + throw new DdlException("lower_case must be true or false, got: " + props.get("lower_case")); + } + if (props.containsKey("token_chars") || props.containsKey("custom_token_chars")) { + throw new DdlException("token_chars cannot be used together with mode (gram tokenizer splits by script)"); + } + } + + /** + * Parses an integer property whose value range is {@code [lo, hi]}; returns the default + * {@code dflt} when the property is not set. + * Same range as BE's `gram_scheme.cpp::parse_uint`; out-of-range and non-numeric values report + * the same message. + */ + private static int parseIntInRange(Map props, String key, int dflt, int lo, int hi) + throws DdlException { + if (!props.containsKey(key)) { + return dflt; + } + String raw = props.get(key); + if (!INTEGER_PATTERN.matcher(raw).matches()) { + throw new DdlException(key + " must be an integer in [" + lo + ", " + hi + "], got: " + raw); + } + try { + int value = Integer.parseInt(raw); + if (value < lo || value > hi) { + throw new DdlException(key + " must be an integer in [" + lo + ", " + hi + "], got: " + raw); + } + return value; + } catch (NumberFormatException e) { + throw new DdlException(key + " must be an integer in [" + lo + ", " + hi + "], got: " + raw); + } + } + + /** + * Parses a double property, throwing a DdlException that carries the field name on failure. + */ + private static double parseDouble(String value, String key) throws DdlException { + if (!DECIMAL_PATTERN.matcher(value).matches()) { + throw new DdlException(key + " must be a decimal number, got: " + value); + } + try { + return Double.parseDouble(value); + } catch (NumberFormatException e) { + throw new DdlException(key + " must be a number, got: " + value); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java index cc427813b76824..ffa215567a770e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java @@ -474,6 +474,34 @@ public Index translateToCatalogStyle() { comment); } + /** + * Syncs the index properties (possibly completed with defaults by {@link #checkColumn}) back into + * an already materialized {@link Index}. + * + *

On the ADD INDEX / standalone CREATE INDEX path, {@code CreateIndexOp#validate} + * materializes the {@link Index} through {@link #translateToCatalogStyle()} before + * {@code checkColumn} runs, while gram-family defaults such as {@code support_phrase=false} are + * only written into the IndexDefinition properties by {@code checkColumn} + * ({@code InvertedIndexUtil#applyGramFamilyIndexDefaults}); without this write-back they would be + * lost from the index that is finally persisted. The CREATE TABLE path runs checkColumn before + * translate and is unaffected. + * + *

Merge rule: a key present on the IndexDefinition side overrides the Index side, and a key + * only the Index side has is kept -- the latter is a default filled in by the {@link Index} + * constructor's general rules (for example {@code lower_case=true} for a parser index) and must + * not be lost to this write-back. The passed instance is modified in place, because the caller + * may already hold the same Index reference. + */ + public void applyPropertiesTo(Index index) { + if (index == null || properties == null || properties.isEmpty()) { + return; + } + Map merged = index.getProperties() == null + ? new HashMap<>() : new HashMap<>(index.getProperties()); + merged.putAll(properties); + index.setProperties(merged); + } + public List getPartitionNames() { return partitionNames == null ? Lists.newArrayList() : partitionNames.getPartitionNames(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/GramDdlValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/GramDdlValidationTest.java new file mode 100644 index 00000000000000..05620b6def8fe7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/GramDdlValidationTest.java @@ -0,0 +1,331 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.indexpolicy; + +import org.apache.doris.analysis.InvertedIndexUtil; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.info.IndexType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.UserException; +import org.apache.doris.nereids.trees.plans.commands.info.CreateIndexOp; +import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; + +import com.google.common.collect.Lists; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * Task 14: the analyzer graph constraint for gram-family analyzers (R31: an ngram tokenizer with + * mode may not stack any token filter) and the index property constraints (a gram-family index + * only works on SNII, does not support phrases, forbids an index-level char_filter, and defaults + * support_phrase to false when it is not given explicitly). + */ +public class GramDdlValidationTest { + + private IndexPolicyMgr manager; + + @BeforeEach + public void setUp() { + manager = new IndexPolicyMgr(); + // gram_sparse_tok: a custom TOKENIZER, ngram + mode=sparse (gram family). + manager.replayCreateIndexPolicy(policy(1L, "gram_sparse_tok", IndexPolicyTypeEnum.TOKENIZER, + Map.of("type", "ngram", "mode", "sparse"))); + // gram_sparse: a valid gram-family analyzer referencing only the tokenizer, no token_filter. + manager.replayCreateIndexPolicy(policy(2L, "gram_sparse", IndexPolicyTypeEnum.ANALYZER, + Map.of("tokenizer", "gram_sparse_tok"))); + // plain_tok / plain: legacy ngram (min_gram/max_gram, no mode), not gram family. + manager.replayCreateIndexPolicy(policy(3L, "plain_tok", IndexPolicyTypeEnum.TOKENIZER, + Map.of("type", "ngram", "min_gram", "2", "max_gram", "3"))); + manager.replayCreateIndexPolicy(policy(4L, "plain", IndexPolicyTypeEnum.ANALYZER, + Map.of("tokenizer", "plain_tok"))); + // gram_dense_tok: ngram + mode=dense, to check that R31 applies to dense as well + // (stricter than the brief). + manager.replayCreateIndexPolicy(policy(5L, "gram_dense_tok", IndexPolicyTypeEnum.TOKENIZER, + Map.of("type", "ngram", "mode", "dense"))); + } + + // ---------------- resolveGramTokenizerMode ---------------- + + @Test + public void testResolveGramMode() { + Assertions.assertEquals(Optional.of("sparse"), manager.resolveGramTokenizerMode("gram_sparse")); + Assertions.assertEquals(Optional.empty(), manager.resolveGramTokenizerMode("plain")); + Assertions.assertEquals(Optional.empty(), manager.resolveGramTokenizerMode("english")); + Assertions.assertEquals(Optional.empty(), manager.resolveGramTokenizerMode("no_such_analyzer")); + } + + // ------------- validateAnalyzerGraphLocked (R31, triggered via createIndexPolicy) ------------- + + @Test + public void testLowercaseFilterRejectedWithSparseMode() { + Map props = new HashMap<>(); + props.put("tokenizer", "gram_sparse_tok"); + props.put("token_filter", "lowercase"); + UserException e = Assertions.assertThrows(UserException.class, + () -> manager.createIndexPolicy(false, "bad_analyzer", IndexPolicyTypeEnum.ANALYZER, props)); + Assertions.assertTrue(e.getMessage().contains("lowercase token filter cannot be combined"), + e.getMessage()); + } + + @Test + public void testOtherFilterRejectedWithSparseMode() { + // Any token filter other than lowercase (the built-in asciifolding here) must be rejected + // too: R31 requires a gram-family analyzer to be a bare tokenizer, not just free of + // lowercase. + Map props = new HashMap<>(); + props.put("tokenizer", "gram_sparse_tok"); + props.put("token_filter", "asciifolding"); + UserException e = Assertions.assertThrows(UserException.class, + () -> manager.createIndexPolicy(false, "bad_analyzer2", IndexPolicyTypeEnum.ANALYZER, props)); + Assertions.assertTrue(e.getMessage().contains("cannot be combined"), e.getMessage()); + } + + @Test + public void testLowercaseFilterRejectedWhenNotFirstInChain() { + // When lowercase is not first in the chain, the message must still be the more specific + // lowercase hint: what the user has to change is "switch to the tokenizer's own + // lower_case", not remove asciifolding first. + Map props = new HashMap<>(); + props.put("tokenizer", "gram_sparse_tok"); + props.put("token_filter", "asciifolding,lowercase"); + UserException e = Assertions.assertThrows(UserException.class, + () -> manager.createIndexPolicy(false, "bad_analyzer4", IndexPolicyTypeEnum.ANALYZER, props)); + Assertions.assertTrue(e.getMessage().contains("lowercase token filter cannot be combined"), + e.getMessage()); + } + + @Test + public void testLowercaseFilterRejectedWithDenseMode() { + // The corrected ruling R31 is stricter than the brief: dense mode must be rejected too, + // rather than let through as in the brief's draft. + Map props = new HashMap<>(); + props.put("tokenizer", "gram_dense_tok"); + props.put("token_filter", "lowercase"); + UserException e = Assertions.assertThrows(UserException.class, + () -> manager.createIndexPolicy(false, "bad_analyzer3", IndexPolicyTypeEnum.ANALYZER, props)); + Assertions.assertTrue(e.getMessage().contains("lowercase token filter cannot be combined"), + e.getMessage()); + } + + @Test + public void testCharFilterRejectedWithSparseMode() { + // BE decides the gram family in custom_analyzer.cpp with + // `!char_filter_configs.empty() || !token_filter_configs.empty()`: ANY filter, char filter + // included, drops the gram scheme. FE checking only token filters lets an analyzer through + // that FE then constrains like a gram index (SNII required, support_phrase forced to + // false) while BE builds an index that can never accelerate LIKE/REGEXP -- a silently + // degraded index with no warning anywhere. FE must mirror BE's rule exactly. + Map props = new HashMap<>(); + props.put("tokenizer", "gram_sparse_tok"); + props.put("char_filter", "char_replace"); + UserException e = Assertions.assertThrows(UserException.class, + () -> manager.createIndexPolicy(false, "bad_analyzer_cf", IndexPolicyTypeEnum.ANALYZER, props)); + Assertions.assertTrue(e.getMessage().contains("cannot be combined"), e.getMessage()); + } + + @Test + public void testCharFilterRejectedWithDenseMode() { + Map props = new HashMap<>(); + props.put("tokenizer", "gram_dense_tok"); + props.put("char_filter", "char_replace"); + UserException e = Assertions.assertThrows(UserException.class, + () -> manager.createIndexPolicy(false, "bad_analyzer_cf2", IndexPolicyTypeEnum.ANALYZER, props)); + Assertions.assertTrue(e.getMessage().contains("cannot be combined"), e.getMessage()); + } + + @Test + public void testCharFilterStillAllowedOnNonGramAnalyzer() { + // The rule is scoped to gram-family tokenizers: a legacy ngram analyzer (min_gram/max_gram, + // no mode) may still stack a char filter, so this must not become a blanket rejection. + // This harness has no EditLog, so an analyzer that passes validation goes on to fail in the + // persist step; only the validation outcome is asserted here. + Map props = new HashMap<>(); + props.put("tokenizer", "plain_tok"); + props.put("char_filter", "char_replace"); + try { + manager.createIndexPolicy(false, "plain_with_cf", IndexPolicyTypeEnum.ANALYZER, props); + } catch (UserException e) { + Assertions.fail("a non-gram analyzer must keep its char filter, but validation rejected it: " + + e.getMessage()); + } catch (RuntimeException expectedWithoutEditLog) { + // Reaching the persist step is the proof that validation accepted the analyzer. + } + } + + @Test + public void testGramTokenizerOnlyAnalyzerRemainsValid() throws Exception { + // A gram-family analyzer that is a bare tokenizer (no token_filter at all) must stay + // allowed and must not be caught by R31; "gram_sparse" is exactly that valid shape + // (registered in setUp). + Assertions.assertDoesNotThrow(() -> manager.validateAnalyzerExists("gram_sparse")); + Assertions.assertEquals(Optional.of("sparse"), manager.resolveGramTokenizerMode("gram_sparse")); + } + + // ---------- index property constraints of InvertedIndexUtil.checkInvertedIndexParser ---------- + + @Test + public void testIndexPropertiesForGramAnalyzer() throws Exception { + IndexPolicyMgr mockMgr = gramAnalyzerManager(); + Map props = new HashMap<>(); + props.put("analyzer", "gram_sparse"); + withIndexPolicyManager(mockMgr, () -> Assertions.assertDoesNotThrow( + () -> InvertedIndexUtil.checkInvertedIndexParser("c", PrimitiveType.VARCHAR, props, + TInvertedIndexFileStorageFormat.SNII))); + // When support_phrase is not given explicitly it must be forced to "false", overriding the + // general rule of the Index constructor that "an analyzer implies true". + Assertions.assertEquals("false", props.get("support_phrase")); + } + + @Test + public void testIndexPropertiesRejectExplicitSupportPhraseTrue() throws Exception { + IndexPolicyMgr mockMgr = gramAnalyzerManager(); + Map phrase = new HashMap<>(); + phrase.put("analyzer", "gram_sparse"); + phrase.put("support_phrase", "true"); + withIndexPolicyManager(mockMgr, () -> { + AnalysisException e = Assertions.assertThrows(AnalysisException.class, + () -> InvertedIndexUtil.checkInvertedIndexParser("c", PrimitiveType.VARCHAR, phrase, + TInvertedIndexFileStorageFormat.SNII)); + Assertions.assertTrue(e.getMessage().contains("does not support phrase"), e.getMessage()); + }); + } + + @Test + public void testIndexPropertiesRejectNonSniiStorageFormat() throws Exception { + IndexPolicyMgr mockMgr = gramAnalyzerManager(); + Map v2 = new HashMap<>(); + v2.put("analyzer", "gram_sparse"); + withIndexPolicyManager(mockMgr, () -> { + AnalysisException e = Assertions.assertThrows(AnalysisException.class, + () -> InvertedIndexUtil.checkInvertedIndexParser("c", PrimitiveType.VARCHAR, v2, + TInvertedIndexFileStorageFormat.V2)); + Assertions.assertTrue(e.getMessage().contains("requires inverted_index_storage_format = SNII"), + e.getMessage()); + }); + } + + @Test + public void testIndexPropertiesRejectCharFilterWithGramAnalyzer() throws Exception { + IndexPolicyMgr mockMgr = gramAnalyzerManager(); + Map props = new HashMap<>(); + props.put("analyzer", "gram_sparse"); + props.put("char_filter_type", "char_replace"); + props.put("char_filter_pattern", "-"); + withIndexPolicyManager(mockMgr, () -> { + AnalysisException e = Assertions.assertThrows(AnalysisException.class, + () -> InvertedIndexUtil.checkInvertedIndexParser("c", PrimitiveType.VARCHAR, props, + TInvertedIndexFileStorageFormat.SNII)); + Assertions.assertTrue(e.getMessage().contains("char_filter cannot be used with gram tokenizer"), + e.getMessage()); + }); + } + + /** + * The ADD INDEX / standalone CREATE INDEX path: {@code CreateIndexOp#validate} materializes the + * Index first, and {@code SchemaChangeHandler#processAddIndex} only runs {@code checkColumn} + * afterwards. The gram family's support_phrase=false default is written into the + * IndexDefinition during checkColumn and has to be copied back by + * {@code IndexDefinition#applyPropertiesTo} for the persisted Index to carry it. + * + *

{@code processAddIndex} needs a complete Env/OlapTable to run, so this drives the same + * three steps in its call order (translateToCatalogStyle -> checkColumn -> applyPropertiesTo). + */ + @Test + public void testAddIndexPathKeepsGramSupportPhraseDefault() throws Exception { + IndexPolicyMgr mockMgr = gramAnalyzerManager(); + Map props = new HashMap<>(); + props.put("analyzer", "gram_sparse"); + IndexDefinition indexDef = new IndexDefinition("idx_g", false, Lists.newArrayList("msg"), + "INVERTED", props, ""); + CreateIndexOp createIndexOp = new CreateIndexOp(null, indexDef, true); + withIndexPolicyManager(mockMgr, () -> { + Assertions.assertDoesNotThrow(() -> createIndexOp.validate(null)); + Index index = createIndexOp.getIndex(); + // Materialization happens before checkColumn, so the Index constructor has filled in + // true by the rule "an analyzer implies phrase support" + Assertions.assertEquals("true", index.getProperties().get("support_phrase")); + + // checkColumn inside processAddIndex: the gram-family default lands in the + // IndexDefinition only + indexDef.checkColumn(new Column("msg", PrimitiveType.STRING), KeysType.DUP_KEYS, false, + TInvertedIndexFileStorageFormat.SNII); + Assertions.assertEquals("false", indexDef.getProperties().get("support_phrase")); + + // Only after the write-back does the Index that is actually persisted carry + // support_phrase=false + indexDef.applyPropertiesTo(index); + Assertions.assertEquals("false", index.getProperties().get("support_phrase")); + // The write-back must modify the same instance in place: processAddIndex's caller + // already holds this reference + Assertions.assertSame(index, createIndexOp.getIndex()); + }); + } + + /** + * The write-back only overwrites keys present in the IndexDefinition: a default filled in by + * the Index constructor's general rules that the IndexDefinition side lacks (such as + * lower_case=true for a parser index) must be preserved as is. + */ + @Test + public void testApplyPropertiesToKeepsIndexOnlyDefaults() { + Map props = new HashMap<>(); + props.put("parser", "english"); + IndexDefinition indexDef = new IndexDefinition("idx_p", false, Lists.newArrayList("msg"), + "INVERTED", props, ""); + Index index = new Index(1L, "idx_p", Lists.newArrayList("msg"), + IndexType.INVERTED, props, ""); + Assertions.assertEquals("true", index.getProperties().get("lower_case")); + Assertions.assertEquals("true", index.getProperties().get("support_phrase")); + + indexDef.applyPropertiesTo(index); + Assertions.assertEquals("english", index.getProperties().get("parser")); + Assertions.assertEquals("true", index.getProperties().get("lower_case")); + Assertions.assertEquals("true", index.getProperties().get("support_phrase")); + } + + private static IndexPolicyMgr gramAnalyzerManager() throws Exception { + IndexPolicyMgr mockMgr = Mockito.mock(IndexPolicyMgr.class); + Mockito.when(mockMgr.resolveGramTokenizerMode("gram_sparse")).thenReturn(Optional.of("sparse")); + return mockMgr; + } + + private static void withIndexPolicyManager(IndexPolicyMgr manager, Runnable action) { + Env env = Mockito.mock(Env.class); + Mockito.when(env.getIndexPolicyMgr()).thenReturn(manager); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + action.run(); + } + } + + private static IndexPolicy policy(long id, String name, IndexPolicyTypeEnum type, Map props) { + return new IndexPolicy(id, name, type, new HashMap<>(props)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/PolicyValidatorTests.java b/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/PolicyValidatorTests.java index 1755acec1ccf65..3617fb162a9c1c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/PolicyValidatorTests.java +++ b/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/PolicyValidatorTests.java @@ -671,6 +671,218 @@ private static Index invertedIndex(long id, String key, String name) { return new Index(id, "idx_" + id, List.of("content"), IndexType.INVERTED, Map.of(key, name), ""); } + // NGramTokenizerValidator gram-mode (auto/sparse/dense) Tests + @Test + public void testNGramValidator_GramModeSparse() throws DdlException { + NGramTokenizerValidator validator = new NGramTokenizerValidator(); + Map props = new HashMap<>(); + props.put("type", "ngram"); + props.put("mode", "sparse"); + props.put("min_gram", "3"); + props.put("max_gram", "16"); + props.put("density", "0.25"); + props.put("lower_case", "true"); + validator.validate(props); // does not throw + } + + @Test + public void testNGramValidator_GramModeRejectsBadValues() { + NGramTokenizerValidator validator = new NGramTokenizerValidator(); + Map bad = new HashMap<>(); + bad.put("type", "ngram"); + bad.put("mode", "fuzzy"); + DdlException e1 = Assertions.assertThrows(DdlException.class, () -> validator.validate(bad)); + Assertions.assertTrue(e1.getMessage().contains("mode must be one of")); + + Map noMode = new HashMap<>(); + noMode.put("type", "ngram"); + noMode.put("density", "0.25"); + DdlException e2 = Assertions.assertThrows(DdlException.class, () -> validator.validate(noMode)); + Assertions.assertTrue(e2.getMessage().contains("requires mode")); + + Map badDensity = new HashMap<>(); + badDensity.put("type", "ngram"); + badDensity.put("mode", "sparse"); + badDensity.put("density", "1.5"); + Assertions.assertTrue(Assertions.assertThrows(DdlException.class, () -> validator.validate(badDensity)) + .getMessage().contains("density must be")); + + Map tokenChars = new HashMap<>(); + tokenChars.put("type", "ngram"); + tokenChars.put("mode", "dense"); + tokenChars.put("token_chars", "letter"); + Assertions.assertTrue(Assertions.assertThrows(DdlException.class, () -> validator.validate(tokenChars)) + .getMessage().contains("token_chars cannot be used")); + + Map wideGap = new HashMap<>(); // max-min>1 is allowed once mode is set + wideGap.put("type", "ngram"); + wideGap.put("mode", "sparse"); + wideGap.put("min_gram", "3"); + wideGap.put("max_gram", "24"); + Assertions.assertDoesNotThrow(() -> validator.validate(wideGap)); + } + + @Test + public void testNGramValidator_GramModeRejectsEmptyMode() { + // BE treats an empty mode as legacy, but FE validation must reject an empty mode string + // already at DDL time. + NGramTokenizerValidator validator = new NGramTokenizerValidator(); + Map emptyMode = new HashMap<>(); + emptyMode.put("type", "ngram"); + emptyMode.put("mode", ""); + DdlException e = Assertions.assertThrows(DdlException.class, () -> validator.validate(emptyMode)); + Assertions.assertTrue(e.getMessage().contains("mode must be one of"), e.getMessage()); + // The empty value must be recognizable in the message, not leave a blank "got: " + Assertions.assertTrue(e.getMessage().contains("got: '' (empty)"), e.getMessage()); + } + + private static Map sparseGramProps() { + Map props = new HashMap<>(); + props.put("type", "ngram"); + props.put("mode", "sparse"); + return props; + } + + private static String assertGramPropRejected(Map props) { + NGramTokenizerValidator validator = new NGramTokenizerValidator(); + return Assertions.assertThrows(DdlException.class, () -> validator.validate(props)).getMessage(); + } + + /** + * The value domains of the gram-family parameters must match BE's + * `gram_scheme.cpp::from_properties`: + * min_gram in [1, 64], max_gram in [1, 256], density in [0.001, 1]. + * Letting an out-of-range value through in FE only defers the error to a BE InvalidArgument at + * write time. + */ + @Test + public void testNGramValidator_GramModeValueDomainsMirrorBe() { + NGramTokenizerValidator validator = new NGramTokenizerValidator(); + + Map maxGramTooBig = sparseGramProps(); + maxGramTooBig.put("max_gram", "257"); // BE's upper bound is 256 + String maxGramMessage = assertGramPropRejected(maxGramTooBig); + Assertions.assertTrue(maxGramMessage.contains("max_gram must be an integer in [1, 256]"), maxGramMessage); + + Map minGramTooBig = sparseGramProps(); + minGramTooBig.put("min_gram", "65"); // BE's upper bound is 64 + String minGramMessage = assertGramPropRejected(minGramTooBig); + Assertions.assertTrue(minGramMessage.contains("min_gram must be an integer in [1, 64]"), minGramMessage); + + Map gramAtBound = sparseGramProps(); // the bounds themselves must pass + gramAtBound.put("min_gram", "64"); + gramAtBound.put("max_gram", "256"); + Assertions.assertDoesNotThrow(() -> validator.validate(gramAtBound)); + + Map densityTooSmall = sparseGramProps(); + densityTooSmall.put("density", "0.0005"); // BE's lower bound is 0.001 (permille) + String densityMessage = assertGramPropRejected(densityTooSmall); + Assertions.assertTrue(densityMessage.contains("density must be in [0.001, 1]"), densityMessage); + + Map densityAtBound = sparseGramProps(); + densityAtBound.put("density", "0.001"); + Assertions.assertDoesNotThrow(() -> validator.validate(densityAtBound)); + + // stop_gram_df is not a property of this tokenizer: nothing ever consumed it, so the + // concept was removed rather than left as a knob that quietly does nothing. It must now be + // rejected as an unknown property, not accepted and ignored. + Map stopGramDf = sparseGramProps(); + stopGramDf.put("stop_gram_df", "0.10"); + String stopGramDfMessage = assertGramPropRejected(stopGramDf); + Assertions.assertTrue(stopGramDfMessage.contains("stop_gram_df"), stopGramDfMessage); + + Map badLowerCase = sparseGramProps(); + badLowerCase.put("lower_case", "yes"); + String lowerCaseMessage = assertGramPropRejected(badLowerCase); + Assertions.assertTrue(lowerCaseMessage.contains("lower_case must be true or false"), lowerCaseMessage); + + Map inverted = sparseGramProps(); // min <= max holds with mode too + inverted.put("min_gram", "5"); + inverted.put("max_gram", "4"); + String invertedMessage = assertGramPropRejected(inverted); + Assertions.assertTrue(invertedMessage.contains("min_gram (5) must be <= max_gram (4)"), invertedMessage); + } + + /** + * Integer gram properties must be spelled in ASCII, the same way the decimal ones must. + * `Integer.parseInt` resolves any Unicode decimal digit through `Character.digit`, so a + * full-width spelling would pass DDL validation here and then be rejected by BE's + * `gram_scheme.cpp::parse_uint`, which uses `strtol` and only accepts ASCII digits. The result + * would be a CREATE that succeeds and a load that fails with an opaque analyzer error, so FE + * has to reject the non-portable spelling up front. + */ + @Test + public void testNGramValidator_GramIntegerPropsRejectNonAsciiDigits() { + // Spelled as escapes so the assertion does not depend on the source file's encoding. + Map fullWidthMin = sparseGramProps(); + fullWidthMin.put("min_gram", "3"); // FULLWIDTH DIGIT THREE + String minMessage = assertGramPropRejected(fullWidthMin); + Assertions.assertTrue(minMessage.contains("min_gram must be an integer in [1, 64]"), minMessage); + + Map fullWidthMax = sparseGramProps(); + fullWidthMax.put("max_gram", "16"); // FULLWIDTH ONE, FULLWIDTH SIX + String maxMessage = assertGramPropRejected(fullWidthMax); + Assertions.assertTrue(maxMessage.contains("max_gram must be an integer in [1, 256]"), maxMessage); + + // Arabic-Indic digits reach Integer.parseInt through the same Character.digit path. + Map arabicIndic = sparseGramProps(); + arabicIndic.put("min_gram", "٣"); // ARABIC-INDIC DIGIT THREE + String arabicMessage = assertGramPropRejected(arabicIndic); + Assertions.assertTrue(arabicMessage.contains("min_gram must be an integer in [1, 64]"), arabicMessage); + + // The portable ASCII spellings, including an explicit sign that strtol also accepts, stay + // valid -- this rule rejects non-ASCII, it does not narrow the accepted number syntax. + NGramTokenizerValidator validator = new NGramTokenizerValidator(); + Map ascii = sparseGramProps(); + ascii.put("min_gram", "3"); + ascii.put("max_gram", "+16"); + Assertions.assertDoesNotThrow(() -> validator.validate(ascii)); + } + + /** + * The mode value is neither trimmed nor case-folded: BE's `from_properties` compares strings + * exactly, so if FE accepted " Sparse " it would be persisted verbatim and only fail with a BE + * InvalidArgument at write time. + * This pins down the "FE rejects outright" ruling (the alternative would be for FE to normalize + * before persisting, which this implementation does not do). + */ + @Test + public void testNGramValidator_GramModeRejectsUntrimmedAndMixedCase() { + Map padded = new HashMap<>(); + padded.put("type", "ngram"); + padded.put("mode", " Sparse "); + String message = assertGramPropRejected(padded); + Assertions.assertTrue(message.contains("mode must be one of"), message); + Assertions.assertTrue(message.contains("got: ' Sparse '"), message); + + Map upper = new HashMap<>(); + upper.put("type", "ngram"); + upper.put("mode", "SPARSE"); + String upperMessage = assertGramPropRejected(upper); + Assertions.assertTrue(upperMessage.contains("mode must be one of"), upperMessage); + } + + @Test + public void testNGramValidator_GramDecimalPropertiesHavePortableSyntax() { + NGramTokenizerValidator validator = new NGramTokenizerValidator(); + for (String key : new String[] {"density"}) { + // Policies are persisted verbatim and parsed by BE. Java-only suffixes and + // implicit trimming must not defer an accepted DDL's failure to data loading. + for (String value : new String[] {"0.25f", "0.25D", "0.25 ", " 0.25", "0.25\t", + "0x1p-2", "NaN", "Infinity", "", ".", "1e", "0.25"}) { + Map props = sparseGramProps(); + props.put(key, value); + Assertions.assertThrows(DdlException.class, () -> validator.validate(props), + key + "=" + value); + } + for (String value : new String[] {"0.25", ".25", "1.", "+0.25", "2.5e-1", "0.001", "1"}) { + Map props = sparseGramProps(); + props.put(key, value); + Assertions.assertDoesNotThrow(() -> validator.validate(props), key + "=" + value); + } + } + } + // StandardTokenizerValidator Tests @Test public void testStandardTokenizerValidator_ValidProperties() throws Exception { From df5845841abb53870f85f1502a218ad0daa181e9 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Thu, 10 Sep 2026 20:55:19 +0800 Subject: [PATCH 06/36] [test](regression) gram index suites: parity, recall, policy recovery, inheritance, stop-gram, compaction, schema change Seven suites under inverted_index_p0/gram, all comparing LIKE/REGEXP with the index enabled and disabled on the same table with the SQL cache and the condition cache off: - test_gram_regexp_like: 139 REGEXP/RLIKE/LIKE/NOT/compound queries before and after DELETE, across rowsets and three coexisting indexes on one column, with the profile proving the index pruned rows; - test_gram_pattern_recall, test_gram_policy_recovery, test_gram_metadata_inherit (CREATE INDEX / BUILD INDEX inherit the scheme); - test_gram_stop_gram: the same rows as one segment above the row floor and as five below it, identical answers, the dropped posting lists visible as bytes, and the answers holding with the cost gate disabled; - test_gram_compaction and test_gram_schema_change: the answers and the pruning survive a full compaction, a light column add and drop, wider value and key types, a longer VARCHAR and VARCHAR -> STRING on the indexed column, a column reorder, and loads landing after the changes. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01METuP3aVRn8dfCU62PivnF --- .../gram/test_gram_metadata_inherit.out | 56 + .../gram/test_gram_pattern_recall.out | 1609 ++++++++++++ .../gram/test_gram_policy_recovery.out | 67 + .../gram/test_gram_regexp_like.out | 2315 +++++++++++++++++ .../gram/test_gram_compaction.groovy | 168 ++ .../gram/test_gram_metadata_inherit.groovy | 153 ++ .../gram/test_gram_pattern_recall.groovy | 132 + .../gram/test_gram_policy_recovery.groovy | 112 + .../gram/test_gram_regexp_like.groovy | 373 +++ .../gram/test_gram_schema_change.groovy | 270 ++ .../gram/test_gram_stop_gram.groovy | 192 ++ 11 files changed, 5447 insertions(+) create mode 100644 regression-test/data/inverted_index_p0/gram/test_gram_metadata_inherit.out create mode 100644 regression-test/data/inverted_index_p0/gram/test_gram_pattern_recall.out create mode 100644 regression-test/data/inverted_index_p0/gram/test_gram_policy_recovery.out create mode 100644 regression-test/data/inverted_index_p0/gram/test_gram_regexp_like.out create mode 100644 regression-test/suites/inverted_index_p0/gram/test_gram_compaction.groovy create mode 100644 regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy create mode 100644 regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy create mode 100644 regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy create mode 100644 regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy create mode 100644 regression-test/suites/inverted_index_p0/gram/test_gram_schema_change.groovy create mode 100644 regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy diff --git a/regression-test/data/inverted_index_p0/gram/test_gram_metadata_inherit.out b/regression-test/data/inverted_index_p0/gram/test_gram_metadata_inherit.out new file mode 100644 index 00000000000000..30c8aaab4af3bd --- /dev/null +++ b/regression-test/data/inverted_index_p0/gram/test_gram_metadata_inherit.out @@ -0,0 +1,56 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_dense_false_like -- +0 + +-- !before_dense_false_regexp -- +0 + +-- !before_sparse_false_like -- +0 + +-- !before_sparse_false_regexp -- +0 + +-- !before_dense_true_like -- +0 + +-- !before_dense_true_regexp -- +0 + +-- !before_sparse_true_like -- +0 + +-- !before_sparse_true_regexp -- +0 + +-- !after_dense_false_like -- +0 + +-- !after_dense_false_regexp -- +0 + +-- !after_sparse_false_like -- +0 + +-- !after_sparse_false_regexp -- +0 + +-- !after_dense_true_like -- +0 + +-- !after_dense_true_regexp -- +0 + +-- !after_sparse_true_like -- +0 + +-- !after_sparse_true_regexp -- +0 + +-- !new_index -- +0 +1 + +-- !inherited_and_new -- +0 + diff --git a/regression-test/data/inverted_index_p0/gram/test_gram_pattern_recall.out b/regression-test/data/inverted_index_p0/gram/test_gram_pattern_recall.out new file mode 100644 index 00000000000000..01de562ad49086 --- /dev/null +++ b/regression-test/data/inverted_index_p0/gram/test_gram_pattern_recall.out @@ -0,0 +1,1609 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dense_false_0 -- +0 +1 +23 +5 + +-- !dense_false_1 -- +0 +1 +23 +5 + +-- !dense_false_2 -- +2 +3 +4 + +-- !dense_false_3 -- +2 +3 +4 + +-- !dense_false_4 -- +2 +3 +4 + +-- !dense_false_5 -- +5 + +-- !dense_false_6 -- +5 + +-- !dense_false_7 -- +6 + +-- !dense_false_8 -- +6 + +-- !dense_false_9 -- +7 + +-- !dense_false_10 -- +7 + +-- !dense_false_11 -- +10 +7 +8 +9 + +-- !dense_false_12 -- +10 +7 +8 +9 + +-- !dense_false_13 -- +11 + +-- !dense_false_14 -- +11 + +-- !dense_false_15 -- +10 +11 +12 +14 +15 +16 +17 +18 +21 +23 +24 +25 +33 +35 +5 +6 +7 +8 +9 + +-- !dense_false_16 -- +14 +15 + +-- !dense_false_17 -- +16 +17 + +-- !dense_false_18 -- +16 +17 + +-- !dense_false_19 -- +16 +17 + +-- !dense_false_20 -- +10 +11 +14 +15 +16 +17 +19 +20 +21 +22 +23 +5 +6 +7 +8 +9 + +-- !dense_false_21 -- +16 +20 + +-- !dense_false_22 -- +21 + +-- !dense_false_23 -- +16 +17 +20 +21 +22 + +-- !dense_false_24 -- +23 + +-- !dense_false_25 -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +21 +23 +24 +25 +32 +33 +35 +5 +6 +7 +8 +9 + +-- !dense_false_26 -- +28 + +-- !dense_false_27 -- +28 + +-- !dense_false_28 -- +34 +35 + +-- !dense_false_nul_like_0 -- +26 +27 +28 +34 +35 + +-- !dense_false_nul_like_1 -- +27 +28 +34 +35 + +-- !dense_false_nul_like_2 -- +28 + +-- !dense_false_reverse_like -- +32 + +-- !dense_false_reverse_regexp -- +12 +19 +30 +33 + +-- !dense_false_null -- +31 + +-- !dense_false_compound -- +16 +17 +21 + +-- !sparse_false_0 -- +0 +1 +23 +5 + +-- !sparse_false_1 -- +0 +1 +23 +5 + +-- !sparse_false_2 -- +2 +3 +4 + +-- !sparse_false_3 -- +2 +3 +4 + +-- !sparse_false_4 -- +2 +3 +4 + +-- !sparse_false_5 -- +5 + +-- !sparse_false_6 -- +5 + +-- !sparse_false_7 -- +6 + +-- !sparse_false_8 -- +6 + +-- !sparse_false_9 -- +7 + +-- !sparse_false_10 -- +7 + +-- !sparse_false_11 -- +10 +7 +8 +9 + +-- !sparse_false_12 -- +10 +7 +8 +9 + +-- !sparse_false_13 -- +11 + +-- !sparse_false_14 -- +11 + +-- !sparse_false_15 -- +10 +11 +12 +14 +15 +16 +17 +18 +21 +23 +24 +25 +33 +35 +5 +6 +7 +8 +9 + +-- !sparse_false_16 -- +14 +15 + +-- !sparse_false_17 -- +16 +17 + +-- !sparse_false_18 -- +16 +17 + +-- !sparse_false_19 -- +16 +17 + +-- !sparse_false_20 -- +10 +11 +14 +15 +16 +17 +19 +20 +21 +22 +23 +5 +6 +7 +8 +9 + +-- !sparse_false_21 -- +16 +20 + +-- !sparse_false_22 -- +21 + +-- !sparse_false_23 -- +16 +17 +20 +21 +22 + +-- !sparse_false_24 -- +23 + +-- !sparse_false_25 -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +21 +23 +24 +25 +32 +33 +35 +5 +6 +7 +8 +9 + +-- !sparse_false_26 -- +28 + +-- !sparse_false_27 -- +28 + +-- !sparse_false_28 -- +34 +35 + +-- !sparse_false_nul_like_0 -- +26 +27 +28 +34 +35 + +-- !sparse_false_nul_like_1 -- +27 +28 +34 +35 + +-- !sparse_false_nul_like_2 -- +28 + +-- !sparse_false_reverse_like -- +32 + +-- !sparse_false_reverse_regexp -- +12 +19 +30 +33 + +-- !sparse_false_null -- +31 + +-- !sparse_false_compound -- +16 +17 +21 + +-- !dense_lc_false_0 -- +0 +1 +23 +5 + +-- !dense_lc_false_1 -- +0 +1 +23 +5 + +-- !dense_lc_false_2 -- +2 +3 +4 + +-- !dense_lc_false_3 -- +2 +3 +4 + +-- !dense_lc_false_4 -- +2 +3 +4 + +-- !dense_lc_false_5 -- +5 + +-- !dense_lc_false_6 -- +5 + +-- !dense_lc_false_7 -- +6 + +-- !dense_lc_false_8 -- +6 + +-- !dense_lc_false_9 -- +7 + +-- !dense_lc_false_10 -- +7 + +-- !dense_lc_false_11 -- +10 +7 +8 +9 + +-- !dense_lc_false_12 -- +10 +7 +8 +9 + +-- !dense_lc_false_13 -- +11 + +-- !dense_lc_false_14 -- +11 + +-- !dense_lc_false_15 -- +10 +11 +12 +14 +15 +16 +17 +18 +21 +23 +24 +25 +33 +35 +5 +6 +7 +8 +9 + +-- !dense_lc_false_16 -- +14 +15 + +-- !dense_lc_false_17 -- +16 +17 + +-- !dense_lc_false_18 -- +16 +17 + +-- !dense_lc_false_19 -- +16 +17 + +-- !dense_lc_false_20 -- +10 +11 +14 +15 +16 +17 +19 +20 +21 +22 +23 +5 +6 +7 +8 +9 + +-- !dense_lc_false_21 -- +16 +20 + +-- !dense_lc_false_22 -- +21 + +-- !dense_lc_false_23 -- +16 +17 +20 +21 +22 + +-- !dense_lc_false_24 -- +23 + +-- !dense_lc_false_25 -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +21 +23 +24 +25 +32 +33 +35 +5 +6 +7 +8 +9 + +-- !dense_lc_false_26 -- +28 + +-- !dense_lc_false_27 -- +28 + +-- !dense_lc_false_28 -- +34 +35 + +-- !dense_lc_false_nul_like_0 -- +26 +27 +28 +34 +35 + +-- !dense_lc_false_nul_like_1 -- +27 +28 +34 +35 + +-- !dense_lc_false_nul_like_2 -- +28 + +-- !dense_lc_false_reverse_like -- +32 + +-- !dense_lc_false_reverse_regexp -- +12 +19 +30 +33 + +-- !dense_lc_false_null -- +31 + +-- !dense_lc_false_compound -- +16 +17 +21 + +-- !sparse_lc_false_0 -- +0 +1 +23 +5 + +-- !sparse_lc_false_1 -- +0 +1 +23 +5 + +-- !sparse_lc_false_2 -- +2 +3 +4 + +-- !sparse_lc_false_3 -- +2 +3 +4 + +-- !sparse_lc_false_4 -- +2 +3 +4 + +-- !sparse_lc_false_5 -- +5 + +-- !sparse_lc_false_6 -- +5 + +-- !sparse_lc_false_7 -- +6 + +-- !sparse_lc_false_8 -- +6 + +-- !sparse_lc_false_9 -- +7 + +-- !sparse_lc_false_10 -- +7 + +-- !sparse_lc_false_11 -- +10 +7 +8 +9 + +-- !sparse_lc_false_12 -- +10 +7 +8 +9 + +-- !sparse_lc_false_13 -- +11 + +-- !sparse_lc_false_14 -- +11 + +-- !sparse_lc_false_15 -- +10 +11 +12 +14 +15 +16 +17 +18 +21 +23 +24 +25 +33 +35 +5 +6 +7 +8 +9 + +-- !sparse_lc_false_16 -- +14 +15 + +-- !sparse_lc_false_17 -- +16 +17 + +-- !sparse_lc_false_18 -- +16 +17 + +-- !sparse_lc_false_19 -- +16 +17 + +-- !sparse_lc_false_20 -- +10 +11 +14 +15 +16 +17 +19 +20 +21 +22 +23 +5 +6 +7 +8 +9 + +-- !sparse_lc_false_21 -- +16 +20 + +-- !sparse_lc_false_22 -- +21 + +-- !sparse_lc_false_23 -- +16 +17 +20 +21 +22 + +-- !sparse_lc_false_24 -- +23 + +-- !sparse_lc_false_25 -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +21 +23 +24 +25 +32 +33 +35 +5 +6 +7 +8 +9 + +-- !sparse_lc_false_26 -- +28 + +-- !sparse_lc_false_27 -- +28 + +-- !sparse_lc_false_28 -- +34 +35 + +-- !sparse_lc_false_nul_like_0 -- +26 +27 +28 +34 +35 + +-- !sparse_lc_false_nul_like_1 -- +27 +28 +34 +35 + +-- !sparse_lc_false_nul_like_2 -- +28 + +-- !sparse_lc_false_reverse_like -- +32 + +-- !sparse_lc_false_reverse_regexp -- +12 +19 +30 +33 + +-- !sparse_lc_false_null -- +31 + +-- !sparse_lc_false_compound -- +16 +17 +21 + +-- !dense_true_0 -- +0 +1 +23 +5 + +-- !dense_true_1 -- +0 +1 +23 +5 + +-- !dense_true_2 -- +2 +3 +4 + +-- !dense_true_3 -- +2 +3 +4 + +-- !dense_true_4 -- +2 +3 +4 + +-- !dense_true_5 -- +5 + +-- !dense_true_6 -- +5 + +-- !dense_true_7 -- +6 + +-- !dense_true_8 -- +6 + +-- !dense_true_9 -- +7 + +-- !dense_true_10 -- +7 + +-- !dense_true_11 -- +10 +7 +8 +9 + +-- !dense_true_12 -- +10 +7 +8 +9 + +-- !dense_true_13 -- +11 + +-- !dense_true_14 -- +11 + +-- !dense_true_15 -- +10 +11 +12 +14 +15 +16 +17 +18 +21 +23 +24 +25 +33 +35 +5 +6 +7 +8 +9 + +-- !dense_true_16 -- +14 +15 + +-- !dense_true_17 -- +16 +17 + +-- !dense_true_18 -- +16 +17 + +-- !dense_true_19 -- +16 +17 + +-- !dense_true_20 -- +10 +11 +14 +15 +16 +17 +19 +20 +21 +22 +23 +5 +6 +7 +8 +9 + +-- !dense_true_21 -- +16 +20 + +-- !dense_true_22 -- +21 + +-- !dense_true_23 -- +16 +17 +20 +21 +22 + +-- !dense_true_24 -- +23 + +-- !dense_true_25 -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +21 +23 +24 +25 +32 +33 +35 +5 +6 +7 +8 +9 + +-- !dense_true_26 -- +28 + +-- !dense_true_27 -- +28 + +-- !dense_true_28 -- +34 +35 + +-- !dense_true_nul_like_0 -- +26 +27 +28 +34 +35 + +-- !dense_true_nul_like_1 -- +27 +28 +34 +35 + +-- !dense_true_nul_like_2 -- +28 + +-- !dense_true_reverse_like -- +32 + +-- !dense_true_reverse_regexp -- +12 +19 +30 +33 + +-- !dense_true_null -- +31 + +-- !dense_true_compound -- +16 +17 +21 + +-- !sparse_true_0 -- +0 +1 +23 +5 + +-- !sparse_true_1 -- +0 +1 +23 +5 + +-- !sparse_true_2 -- +2 +3 +4 + +-- !sparse_true_3 -- +2 +3 +4 + +-- !sparse_true_4 -- +2 +3 +4 + +-- !sparse_true_5 -- +5 + +-- !sparse_true_6 -- +5 + +-- !sparse_true_7 -- +6 + +-- !sparse_true_8 -- +6 + +-- !sparse_true_9 -- +7 + +-- !sparse_true_10 -- +7 + +-- !sparse_true_11 -- +10 +7 +8 +9 + +-- !sparse_true_12 -- +10 +7 +8 +9 + +-- !sparse_true_13 -- +11 + +-- !sparse_true_14 -- +11 + +-- !sparse_true_15 -- +10 +11 +12 +14 +15 +16 +17 +18 +21 +23 +24 +25 +33 +35 +5 +6 +7 +8 +9 + +-- !sparse_true_16 -- +14 +15 + +-- !sparse_true_17 -- +16 +17 + +-- !sparse_true_18 -- +16 +17 + +-- !sparse_true_19 -- +16 +17 + +-- !sparse_true_20 -- +10 +11 +14 +15 +16 +17 +19 +20 +21 +22 +23 +5 +6 +7 +8 +9 + +-- !sparse_true_21 -- +16 +20 + +-- !sparse_true_22 -- +21 + +-- !sparse_true_23 -- +16 +17 +20 +21 +22 + +-- !sparse_true_24 -- +23 + +-- !sparse_true_25 -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +21 +23 +24 +25 +32 +33 +35 +5 +6 +7 +8 +9 + +-- !sparse_true_26 -- +28 + +-- !sparse_true_27 -- +28 + +-- !sparse_true_28 -- +34 +35 + +-- !sparse_true_nul_like_0 -- +26 +27 +28 +34 +35 + +-- !sparse_true_nul_like_1 -- +27 +28 +34 +35 + +-- !sparse_true_nul_like_2 -- +28 + +-- !sparse_true_reverse_like -- +32 + +-- !sparse_true_reverse_regexp -- +12 +19 +30 +33 + +-- !sparse_true_null -- +31 + +-- !sparse_true_compound -- +16 +17 +21 + +-- !dense_lc_true_0 -- +0 +1 +23 +5 + +-- !dense_lc_true_1 -- +0 +1 +23 +5 + +-- !dense_lc_true_2 -- +2 +3 +4 + +-- !dense_lc_true_3 -- +2 +3 +4 + +-- !dense_lc_true_4 -- +2 +3 +4 + +-- !dense_lc_true_5 -- +5 + +-- !dense_lc_true_6 -- +5 + +-- !dense_lc_true_7 -- +6 + +-- !dense_lc_true_8 -- +6 + +-- !dense_lc_true_9 -- +7 + +-- !dense_lc_true_10 -- +7 + +-- !dense_lc_true_11 -- +10 +7 +8 +9 + +-- !dense_lc_true_12 -- +10 +7 +8 +9 + +-- !dense_lc_true_13 -- +11 + +-- !dense_lc_true_14 -- +11 + +-- !dense_lc_true_15 -- +10 +11 +12 +14 +15 +16 +17 +18 +21 +23 +24 +25 +33 +35 +5 +6 +7 +8 +9 + +-- !dense_lc_true_16 -- +14 +15 + +-- !dense_lc_true_17 -- +16 +17 + +-- !dense_lc_true_18 -- +16 +17 + +-- !dense_lc_true_19 -- +16 +17 + +-- !dense_lc_true_20 -- +10 +11 +14 +15 +16 +17 +19 +20 +21 +22 +23 +5 +6 +7 +8 +9 + +-- !dense_lc_true_21 -- +16 +20 + +-- !dense_lc_true_22 -- +21 + +-- !dense_lc_true_23 -- +16 +17 +20 +21 +22 + +-- !dense_lc_true_24 -- +23 + +-- !dense_lc_true_25 -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +21 +23 +24 +25 +32 +33 +35 +5 +6 +7 +8 +9 + +-- !dense_lc_true_26 -- +28 + +-- !dense_lc_true_27 -- +28 + +-- !dense_lc_true_28 -- +34 +35 + +-- !dense_lc_true_nul_like_0 -- +26 +27 +28 +34 +35 + +-- !dense_lc_true_nul_like_1 -- +27 +28 +34 +35 + +-- !dense_lc_true_nul_like_2 -- +28 + +-- !dense_lc_true_reverse_like -- +32 + +-- !dense_lc_true_reverse_regexp -- +12 +19 +30 +33 + +-- !dense_lc_true_null -- +31 + +-- !dense_lc_true_compound -- +16 +17 +21 + +-- !sparse_lc_true_0 -- +0 +1 +23 +5 + +-- !sparse_lc_true_1 -- +0 +1 +23 +5 + +-- !sparse_lc_true_2 -- +2 +3 +4 + +-- !sparse_lc_true_3 -- +2 +3 +4 + +-- !sparse_lc_true_4 -- +2 +3 +4 + +-- !sparse_lc_true_5 -- +5 + +-- !sparse_lc_true_6 -- +5 + +-- !sparse_lc_true_7 -- +6 + +-- !sparse_lc_true_8 -- +6 + +-- !sparse_lc_true_9 -- +7 + +-- !sparse_lc_true_10 -- +7 + +-- !sparse_lc_true_11 -- +10 +7 +8 +9 + +-- !sparse_lc_true_12 -- +10 +7 +8 +9 + +-- !sparse_lc_true_13 -- +11 + +-- !sparse_lc_true_14 -- +11 + +-- !sparse_lc_true_15 -- +10 +11 +12 +14 +15 +16 +17 +18 +21 +23 +24 +25 +33 +35 +5 +6 +7 +8 +9 + +-- !sparse_lc_true_16 -- +14 +15 + +-- !sparse_lc_true_17 -- +16 +17 + +-- !sparse_lc_true_18 -- +16 +17 + +-- !sparse_lc_true_19 -- +16 +17 + +-- !sparse_lc_true_20 -- +10 +11 +14 +15 +16 +17 +19 +20 +21 +22 +23 +5 +6 +7 +8 +9 + +-- !sparse_lc_true_21 -- +16 +20 + +-- !sparse_lc_true_22 -- +21 + +-- !sparse_lc_true_23 -- +16 +17 +20 +21 +22 + +-- !sparse_lc_true_24 -- +23 + +-- !sparse_lc_true_25 -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +21 +23 +24 +25 +32 +33 +35 +5 +6 +7 +8 +9 + +-- !sparse_lc_true_26 -- +28 + +-- !sparse_lc_true_27 -- +28 + +-- !sparse_lc_true_28 -- +34 +35 + +-- !sparse_lc_true_nul_like_0 -- +26 +27 +28 +34 +35 + +-- !sparse_lc_true_nul_like_1 -- +27 +28 +34 +35 + +-- !sparse_lc_true_nul_like_2 -- +28 + +-- !sparse_lc_true_reverse_like -- +32 + +-- !sparse_lc_true_reverse_regexp -- +12 +19 +30 +33 + +-- !sparse_lc_true_null -- +31 + +-- !sparse_lc_true_compound -- +16 +17 +21 + diff --git a/regression-test/data/inverted_index_p0/gram/test_gram_policy_recovery.out b/regression-test/data/inverted_index_p0/gram/test_gram_policy_recovery.out new file mode 100644 index 00000000000000..387ef025ccd926 --- /dev/null +++ b/regression-test/data/inverted_index_p0/gram/test_gram_policy_recovery.out @@ -0,0 +1,67 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dense3_tokens -- +[{\n "token": "abc"\n }, {\n "token": "bcd"\n }, {\n "token": "cde"\n }, {\n "token": "def"\n }] + +-- !dense4_tokens -- +[{\n "token": "abcd"\n }, {\n "token": "bcde"\n }, {\n "token": "cdef"\n }] + +-- !recovered_false_like -- +1 +2 + +-- !recovered_false_regexp -- +1 +2 + +-- !recovered_true_like -- +1 +2 + +-- !recovered_true_regexp -- +1 +2 + +-- !mixed_false_like -- +1 +2 +5 + +-- !mixed_false_regexp -- +1 +2 +5 + +-- !mixed_false_short_like -- +1 +2 +5 +6 + +-- !mixed_false_short_regexp -- +1 +2 +5 +6 + +-- !mixed_true_like -- +1 +2 +5 + +-- !mixed_true_regexp -- +1 +2 +5 + +-- !mixed_true_short_like -- +1 +2 +5 +6 + +-- !mixed_true_short_regexp -- +1 +2 +5 +6 + diff --git a/regression-test/data/inverted_index_p0/gram/test_gram_regexp_like.out b/regression-test/data/inverted_index_p0/gram/test_gram_regexp_like.out new file mode 100644 index 00000000000000..6a325ee2f3aa20 --- /dev/null +++ b/regression-test/data/inverted_index_p0/gram/test_gram_regexp_like.out @@ -0,0 +1,2315 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !regexp_0_idx_true -- +1 + +-- !rlike_0_idx_true -- +1 + +-- !notregexp_0_idx_true -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_0_idx_true -- + +-- !regexp_1_idx_true -- + +-- !rlike_1_idx_true -- + +-- !notregexp_1_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_1_idx_true -- + +-- !regexp_2_idx_true -- +1 +13 + +-- !rlike_2_idx_true -- +1 +13 + +-- !notregexp_2_idx_true -- +10 +11 +12 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_2_idx_true -- +13 + +-- !regexp_3_idx_true -- +2 + +-- !rlike_3_idx_true -- +2 + +-- !notregexp_3_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_3_idx_true -- + +-- !regexp_4_idx_true -- + +-- !rlike_4_idx_true -- + +-- !notregexp_4_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_4_idx_true -- + +-- !regexp_5_idx_true -- +4 +8 + +-- !rlike_5_idx_true -- +4 +8 + +-- !notregexp_5_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +6 +7 +9 + +-- !regexp_and_5_idx_true -- +4 +8 + +-- !regexp_6_idx_true -- + +-- !rlike_6_idx_true -- + +-- !notregexp_6_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_6_idx_true -- + +-- !regexp_7_idx_true -- +1 +19 +9 + +-- !rlike_7_idx_true -- +1 +19 +9 + +-- !notregexp_7_idx_true -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +2 +3 +4 +6 +7 +8 + +-- !regexp_and_7_idx_true -- +19 +9 + +-- !regexp_8_idx_true -- +4 + +-- !rlike_8_idx_true -- +4 + +-- !notregexp_8_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !regexp_and_8_idx_true -- +4 + +-- !regexp_9_idx_true -- +14 +4 + +-- !rlike_9_idx_true -- +14 +4 + +-- !notregexp_9_idx_true -- +1 +10 +11 +12 +13 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !regexp_and_9_idx_true -- +14 +4 + +-- !regexp_10_idx_true -- +15 + +-- !rlike_10_idx_true -- +15 + +-- !notregexp_10_idx_true -- +1 +10 +11 +12 +13 +14 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_10_idx_true -- +15 + +-- !regexp_11_idx_true -- +1 +13 +15 +2 +7 + +-- !rlike_11_idx_true -- +1 +13 +15 +2 +7 + +-- !notregexp_11_idx_true -- +10 +11 +12 +14 +16 +17 +18 +19 +3 +4 +6 +8 +9 + +-- !regexp_and_11_idx_true -- +13 +15 +7 + +-- !regexp_12_idx_true -- +16 + +-- !rlike_12_idx_true -- +16 + +-- !notregexp_12_idx_true -- +1 +10 +11 +12 +13 +14 +15 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_12_idx_true -- +16 + +-- !regexp_13_idx_true -- +11 + +-- !rlike_13_idx_true -- +11 + +-- !notregexp_13_idx_true -- +1 +10 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_13_idx_true -- +11 + +-- !regexp_14_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !rlike_14_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !notregexp_14_idx_true -- + +-- !regexp_and_14_idx_true -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +4 +6 +7 +8 +9 + +-- !regexp_15_idx_true -- +16 +17 +2 +4 +8 + +-- !rlike_15_idx_true -- +16 +17 +2 +4 +8 + +-- !notregexp_15_idx_true -- +1 +10 +11 +12 +13 +14 +15 +18 +19 +3 +6 +7 +9 + +-- !regexp_and_15_idx_true -- +16 +17 +4 +8 + +-- !regexp_16_idx_true -- + +-- !rlike_16_idx_true -- + +-- !notregexp_16_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_16_idx_true -- + +-- !regexp_17_idx_true -- +1 +11 +12 +13 +4 + +-- !rlike_17_idx_true -- +1 +11 +12 +13 +4 + +-- !notregexp_17_idx_true -- +10 +14 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !regexp_and_17_idx_true -- +11 +12 +13 +4 + +-- !regexp_18_idx_true -- +8 + +-- !rlike_18_idx_true -- +8 + +-- !notregexp_18_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +9 + +-- !regexp_and_18_idx_true -- +8 + +-- !regexp_19_idx_true -- +1 +13 + +-- !rlike_19_idx_true -- +1 +13 + +-- !notregexp_19_idx_true -- +10 +11 +12 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_19_idx_true -- +13 + +-- !regexp_20_idx_true -- +10 + +-- !rlike_20_idx_true -- +10 + +-- !notregexp_20_idx_true -- +1 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_20_idx_true -- +10 + +-- !regexp_21_idx_true -- +1 +15 +7 + +-- !rlike_21_idx_true -- +1 +15 +7 + +-- !notregexp_21_idx_true -- +10 +11 +12 +13 +14 +16 +17 +18 +19 +2 +3 +4 +6 +8 +9 + +-- !regexp_and_21_idx_true -- +15 +7 + +-- !regexp_22_idx_true -- +10 +8 + +-- !rlike_22_idx_true -- +10 +8 + +-- !notregexp_22_idx_true -- +1 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +9 + +-- !regexp_and_22_idx_true -- +10 +8 + +-- !regexp_23_idx_true -- + +-- !rlike_23_idx_true -- + +-- !notregexp_23_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_23_idx_true -- + +-- !regexp_24_idx_true -- +14 +4 + +-- !rlike_24_idx_true -- +14 +4 + +-- !notregexp_24_idx_true -- +1 +10 +11 +12 +13 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !regexp_and_24_idx_true -- +14 +4 + +-- !regexp_25_idx_true -- +8 + +-- !rlike_25_idx_true -- +8 + +-- !notregexp_25_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +9 + +-- !regexp_and_25_idx_true -- +8 + +-- !like_0_idx_true -- +1 +11 +13 + +-- !notlike_0_idx_true -- +10 +12 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !like_1_idx_true -- +1 + +-- !notlike_1_idx_true -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !like_2_idx_true -- +14 +4 + +-- !notlike_2_idx_true -- +1 +10 +11 +12 +13 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !like_3_idx_true -- +15 +7 + +-- !notlike_3_idx_true -- +1 +10 +11 +12 +13 +14 +16 +17 +18 +19 +2 +3 +4 +6 +8 +9 + +-- !like_4_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !notlike_4_idx_true -- + +-- !like_5_idx_true -- +8 + +-- !notlike_5_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +9 + +-- !like_6_idx_true -- +1 + +-- !notlike_6_idx_true -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !like_7_idx_true -- +6 + +-- !notlike_7_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +7 +8 +9 + +-- !like_8_idx_true -- +15 + +-- !notlike_8_idx_true -- +1 +10 +11 +12 +13 +14 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !like_9_idx_true -- +1 +12 + +-- !notlike_9_idx_true -- +10 +11 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !like_10_idx_true -- +15 + +-- !notlike_10_idx_true -- +1 +10 +11 +12 +13 +14 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !like_11_idx_true -- +14 +4 + +-- !notlike_11_idx_true -- +1 +10 +11 +12 +13 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !like_12_idx_true -- +9 + +-- !notlike_12_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 + +-- !compound_0_idx_true -- +1 +11 +13 +14 +4 + +-- !compound_1_idx_true -- +1 +10 +12 +14 +15 +16 +17 +18 +19 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !compound_2_idx_true -- +11 +12 +13 +4 + +-- !compound_3_idx_true -- +1 +11 +13 + +-- !compound_4_idx_true -- +10 +14 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !compound_5_idx_true -- +10 +12 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !compound_6_idx_true -- +1 + +-- !compound_7_idx_true -- +17 + +-- !compound_8_idx_true -- +17 + +-- !count_idx_true -- +5 + +-- !regexp_0_idx_false -- +1 + +-- !rlike_0_idx_false -- +1 + +-- !notregexp_0_idx_false -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_0_idx_false -- + +-- !regexp_1_idx_false -- + +-- !rlike_1_idx_false -- + +-- !notregexp_1_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_1_idx_false -- + +-- !regexp_2_idx_false -- +1 +13 + +-- !rlike_2_idx_false -- +1 +13 + +-- !notregexp_2_idx_false -- +10 +11 +12 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_2_idx_false -- +13 + +-- !regexp_3_idx_false -- +2 + +-- !rlike_3_idx_false -- +2 + +-- !notregexp_3_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_3_idx_false -- + +-- !regexp_4_idx_false -- + +-- !rlike_4_idx_false -- + +-- !notregexp_4_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_4_idx_false -- + +-- !regexp_5_idx_false -- +4 +8 + +-- !rlike_5_idx_false -- +4 +8 + +-- !notregexp_5_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +6 +7 +9 + +-- !regexp_and_5_idx_false -- +4 +8 + +-- !regexp_6_idx_false -- + +-- !rlike_6_idx_false -- + +-- !notregexp_6_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_6_idx_false -- + +-- !regexp_7_idx_false -- +1 +19 +9 + +-- !rlike_7_idx_false -- +1 +19 +9 + +-- !notregexp_7_idx_false -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +2 +3 +4 +6 +7 +8 + +-- !regexp_and_7_idx_false -- +19 +9 + +-- !regexp_8_idx_false -- +4 + +-- !rlike_8_idx_false -- +4 + +-- !notregexp_8_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !regexp_and_8_idx_false -- +4 + +-- !regexp_9_idx_false -- +14 +4 + +-- !rlike_9_idx_false -- +14 +4 + +-- !notregexp_9_idx_false -- +1 +10 +11 +12 +13 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !regexp_and_9_idx_false -- +14 +4 + +-- !regexp_10_idx_false -- +15 + +-- !rlike_10_idx_false -- +15 + +-- !notregexp_10_idx_false -- +1 +10 +11 +12 +13 +14 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_10_idx_false -- +15 + +-- !regexp_11_idx_false -- +1 +13 +15 +2 +7 + +-- !rlike_11_idx_false -- +1 +13 +15 +2 +7 + +-- !notregexp_11_idx_false -- +10 +11 +12 +14 +16 +17 +18 +19 +3 +4 +6 +8 +9 + +-- !regexp_and_11_idx_false -- +13 +15 +7 + +-- !regexp_12_idx_false -- +16 + +-- !rlike_12_idx_false -- +16 + +-- !notregexp_12_idx_false -- +1 +10 +11 +12 +13 +14 +15 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_12_idx_false -- +16 + +-- !regexp_13_idx_false -- +11 + +-- !rlike_13_idx_false -- +11 + +-- !notregexp_13_idx_false -- +1 +10 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_13_idx_false -- +11 + +-- !regexp_14_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !rlike_14_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !notregexp_14_idx_false -- + +-- !regexp_and_14_idx_false -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +4 +6 +7 +8 +9 + +-- !regexp_15_idx_false -- +16 +17 +2 +4 +8 + +-- !rlike_15_idx_false -- +16 +17 +2 +4 +8 + +-- !notregexp_15_idx_false -- +1 +10 +11 +12 +13 +14 +15 +18 +19 +3 +6 +7 +9 + +-- !regexp_and_15_idx_false -- +16 +17 +4 +8 + +-- !regexp_16_idx_false -- + +-- !rlike_16_idx_false -- + +-- !notregexp_16_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_16_idx_false -- + +-- !regexp_17_idx_false -- +1 +11 +12 +13 +4 + +-- !rlike_17_idx_false -- +1 +11 +12 +13 +4 + +-- !notregexp_17_idx_false -- +10 +14 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !regexp_and_17_idx_false -- +11 +12 +13 +4 + +-- !regexp_18_idx_false -- +8 + +-- !rlike_18_idx_false -- +8 + +-- !notregexp_18_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +9 + +-- !regexp_and_18_idx_false -- +8 + +-- !regexp_19_idx_false -- +1 +13 + +-- !rlike_19_idx_false -- +1 +13 + +-- !notregexp_19_idx_false -- +10 +11 +12 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_19_idx_false -- +13 + +-- !regexp_20_idx_false -- +10 + +-- !rlike_20_idx_false -- +10 + +-- !notregexp_20_idx_false -- +1 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_20_idx_false -- +10 + +-- !regexp_21_idx_false -- +1 +15 +7 + +-- !rlike_21_idx_false -- +1 +15 +7 + +-- !notregexp_21_idx_false -- +10 +11 +12 +13 +14 +16 +17 +18 +19 +2 +3 +4 +6 +8 +9 + +-- !regexp_and_21_idx_false -- +15 +7 + +-- !regexp_22_idx_false -- +10 +8 + +-- !rlike_22_idx_false -- +10 +8 + +-- !notregexp_22_idx_false -- +1 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +9 + +-- !regexp_and_22_idx_false -- +10 +8 + +-- !regexp_23_idx_false -- + +-- !rlike_23_idx_false -- + +-- !notregexp_23_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !regexp_and_23_idx_false -- + +-- !regexp_24_idx_false -- +14 +4 + +-- !rlike_24_idx_false -- +14 +4 + +-- !notregexp_24_idx_false -- +1 +10 +11 +12 +13 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !regexp_and_24_idx_false -- +14 +4 + +-- !regexp_25_idx_false -- +8 + +-- !rlike_25_idx_false -- +8 + +-- !notregexp_25_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +9 + +-- !regexp_and_25_idx_false -- +8 + +-- !like_0_idx_false -- +1 +11 +13 + +-- !notlike_0_idx_false -- +10 +12 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !like_1_idx_false -- +1 + +-- !notlike_1_idx_false -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !like_2_idx_false -- +14 +4 + +-- !notlike_2_idx_false -- +1 +10 +11 +12 +13 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !like_3_idx_false -- +15 +7 + +-- !notlike_3_idx_false -- +1 +10 +11 +12 +13 +14 +16 +17 +18 +19 +2 +3 +4 +6 +8 +9 + +-- !like_4_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !notlike_4_idx_false -- + +-- !like_5_idx_false -- +8 + +-- !notlike_5_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +9 + +-- !like_6_idx_false -- +1 + +-- !notlike_6_idx_false -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !like_7_idx_false -- +6 + +-- !notlike_7_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +7 +8 +9 + +-- !like_8_idx_false -- +15 + +-- !notlike_8_idx_false -- +1 +10 +11 +12 +13 +14 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !like_9_idx_false -- +1 +12 + +-- !notlike_9_idx_false -- +10 +11 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !like_10_idx_false -- +15 + +-- !notlike_10_idx_false -- +1 +10 +11 +12 +13 +14 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !like_11_idx_false -- +14 +4 + +-- !notlike_11_idx_false -- +1 +10 +11 +12 +13 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !like_12_idx_false -- +9 + +-- !notlike_12_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 + +-- !compound_0_idx_false -- +1 +11 +13 +14 +4 + +-- !compound_1_idx_false -- +1 +10 +12 +14 +15 +16 +17 +18 +19 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !compound_2_idx_false -- +11 +12 +13 +4 + +-- !compound_3_idx_false -- +1 +11 +13 + +-- !compound_4_idx_false -- +10 +14 +15 +16 +17 +18 +19 +2 +3 +6 +7 +8 +9 + +-- !compound_5_idx_false -- +10 +12 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 + +-- !compound_6_idx_false -- +1 + +-- !compound_7_idx_false -- +17 + +-- !compound_8_idx_false -- +17 + +-- !count_idx_false -- +5 + +-- !profile_q -- +10 + +-- !profile_q_like -- +16 + +-- !after_delete_idx_true -- +11 +13 + +-- !after_delete_idx_false -- +11 +13 + diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_compaction.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_compaction.groovy new file mode 100644 index 00000000000000..ed444beac12726 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_compaction.groovy @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// A gram index has to survive compaction, and until now nothing checked that it does. +// +// Gram indexes are written DOCS_ONLY, and the SNII index-merge fast path refuses any source +// without positions (snii/compaction/eligibility.cpp), so a compaction cannot stitch these +// indexes together -- it rebuilds each one from the merged column data instead. That rebuild +// recomputes everything the segment carries: the gram scheme, the df statistics, the high-df +// digest, and, where it is enabled, which posting lists get dropped. Every one of those is +// derived from the segment's own size, and compaction changes the segment's size, so the +// rebuilt index is not a copy of its inputs and cannot be assumed to behave like them. +// +// The suite therefore checks the property that has to hold either way: the same queries return +// the same rows before and after, and the index is still there afterwards rather than having +// been quietly dropped. Note the table deliberately does NOT set disable_auto_compaction -- +// every existing benchmark and suite for this index does, which is exactly why this path had +// no coverage. +suite("test_gram_compaction", "p0") { + def waitAnalyzerInstalled = { String name -> + def deadline = System.currentTimeMillis() + 180_000 + Exception lastNotFound = null + while (System.currentTimeMillis() < deadline) { + try { + sql """SELECT TOKENIZE('probe', '"analyzer"="${name}"')""" + return + } catch (Exception e) { + if (!e.message.contains("Policy not found")) { + throw e + } + lastNotFound = e + sleep(1000) + } + } + throw new IllegalStateException("analyzer ${name} was not installed on BE", lastNotFound) + } + + def tableName = "test_gram_compaction" + + // The table goes first: a policy still referenced by a table left behind by an earlier run + // cannot be dropped. + sql "DROP TABLE IF EXISTS ${tableName}" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_compact_ana" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_compact_tok" + sql """CREATE INVERTED INDEX TOKENIZER gram_compact_tok PROPERTIES ( + "type"="ngram", "mode"="sparse", "min_gram"="3", "max_gram"="8", "density"="0.5")""" + sql """CREATE INVERTED INDEX ANALYZER gram_compact_ana + PROPERTIES ("tokenizer"="gram_compact_tok")""" + waitAnalyzerInstalled("gram_compact_ana") + + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `msg` text NULL, + INDEX idx_msg (`msg`) USING INVERTED + PROPERTIES('analyzer'='gram_compact_ana', 'support_phrase'='false') + ) ENGINE=OLAP DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES('replication_num'='1', 'inverted_index_storage_format'='SNII')""" + + sql "SET enable_sql_cache=false" + sql "SET enable_condition_cache=false" + + // Several batches, so there are several rowsets for the compaction to merge. Row content + // spans the selectivity range the gate cares about: a marker on every row, one on a few + // hundred, and one on a handful. + def batches = 5 + def perBatch = 800 + for (int b = 0; b < batches; b++) { + def values = [] + for (int i = 0; i < perBatch; i++) { + def id = b * perBatch + i + def rare = (id % 400 == 0) ? " rare_marker_${id}" : "" + def mid = (id % 7 == 0) ? " midfreq_token" : "" + values.add("(${id}, 'shared_prefix_common_text filler_${id}${mid}${rare}')") + } + sql "INSERT INTO ${tableName} VALUES ${values.join(',')}" + } + + def patterns = [ + "like_rare" : "SELECT COUNT(*) FROM ${tableName} WHERE msg LIKE '%rare_marker_%'", + "like_mid" : "SELECT COUNT(*) FROM ${tableName} WHERE msg LIKE '%midfreq_token%'", + "like_common" : "SELECT COUNT(*) FROM ${tableName} WHERE msg LIKE '%shared_prefix_common%'", + "regexp_rare" : "SELECT COUNT(*) FROM ${tableName} WHERE msg REGEXP 'rare_marker_[0-9]+'", + "regexp_mid" : "SELECT COUNT(*) FROM ${tableName} WHERE msg REGEXP 'midfreq_[a-z]+'", + "regexp_absent" : "SELECT COUNT(*) FROM ${tableName} WHERE msg REGEXP 'no_such_token_anywhere'", + "regexp_alt" : "SELECT COUNT(*) FROM ${tableName} WHERE msg REGEXP 'rare_marker_(0|400|800)\\\\b'", + ] + + def runAll = { String label -> + def out = [:] + patterns.each { name, stmt -> out[name] = sql(stmt)[0][0] } + logger.info("${label}: ${out}") + return out + } + + // Tablet statistics reach information_schema asynchronously, so a freshly loaded table can + // report 0 index bytes for a while: poll rather than read once. + def indexBytes = { + def deadline = System.currentTimeMillis() + 180_000 + long bytes = 0L + while (System.currentTimeMillis() < deadline) { + def rows = sql """SELECT INDEX_LENGTH FROM information_schema.tables + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '${tableName}'""" + bytes = rows.isEmpty() ? 0L : (rows[0][0] as long) + if (bytes > 0) { + break + } + sleep(2000) + } + return bytes + } + + // Ground truth first, with the index taken out of the picture entirely. + sql "SET enable_inverted_index_query=false" + def scanned = runAll("full scan") + sql "SET enable_inverted_index_query=true" + + def before = runAll("indexed, before compaction") + scanned.each { name, value -> + assertEquals(value, before[name], "index and scan disagree on ${name} before compaction") + } + def bytesBefore = indexBytes() + logger.info("index bytes before compaction: ${bytesBefore}") + assertTrue(bytesBefore > 0, "the gram index must occupy bytes before compaction") + + trigger_and_wait_compaction(tableName, "full") + + def after = runAll("indexed, after compaction") + before.each { name, value -> + assertEquals(value, after[name], "compaction changed the answer for ${name}") + } + + // The index must have been rebuilt, not dropped: a compaction that quietly produced an + // index-free rowset would still answer every query above correctly, by scanning. + def bytesAfter = indexBytes() + logger.info("index bytes after compaction: ${bytesAfter}") + assertTrue(bytesAfter > 0, + "the gram index must still exist after compaction (bytes before=${bytesBefore}, after=${bytesAfter})") + + // And it must still answer correctly with the index forced off, which pins that the rows + // themselves survived the merge rather than the index merely agreeing with a broken table. + sql "SET enable_inverted_index_query=false" + def scannedAfter = runAll("full scan, after compaction") + sql "SET enable_inverted_index_query=true" + after.each { name, value -> + assertEquals(scannedAfter[name], value, + "index and scan disagree on ${name} after compaction") + } + + sql "DROP TABLE IF EXISTS ${tableName}" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_compact_ana" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_compact_tok" +} diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy new file mode 100644 index 00000000000000..322753075672cd --- /dev/null +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.regex.Pattern + +import org.apache.doris.regression.action.ProfileAction + +suite("test_gram_metadata_inherit", "p0") { + // Cloud does not support the named-index BUILD syntax used here. Keep this case scoped + // to adding one requested index while IndexBuilder inherits the unchanged gram indexes. + if (isCloudMode()) { + return + } + + def waitAnalyzerInstalled = { String name -> + awaitUntil(180) { + try { + sql """SELECT TOKENIZE('abcdefgh', '"analyzer"="${name}"')""" + return true + } catch (Exception e) { + if (!e.message.contains("Policy not found")) { + throw e + } + return false + } + } + } + + sql "DROP TABLE IF EXISTS test_gram_metadata_inherit" + ['dense', 'sparse'].each { name -> + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_inherit_${name}" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_inherit_${name}_tok" + } + sql """CREATE INVERTED INDEX TOKENIZER gram_inherit_dense_tok PROPERTIES ( + "type"="ngram", "mode"="dense", "min_gram"="3")""" + sql """CREATE INVERTED INDEX TOKENIZER gram_inherit_sparse_tok PROPERTIES ( + "type"="ngram", "mode"="sparse", "min_gram"="4", "max_gram"="8", + "density"="1.0")""" + ['dense', 'sparse'].each { name -> + sql """CREATE INVERTED INDEX ANALYZER gram_inherit_${name} + PROPERTIES ("tokenizer"="gram_inherit_${name}_tok")""" + waitAnalyzerInstalled("gram_inherit_${name}") + } + + sql """CREATE TABLE test_gram_metadata_inherit ( + id INT, + dense VARCHAR(128), + sparse VARCHAR(128), + tag VARCHAR(32), + INDEX idx_dense (dense) USING INVERTED PROPERTIES ("analyzer"="gram_inherit_dense"), + INDEX idx_sparse (sparse) USING INVERTED PROPERTIES ("analyzer"="gram_inherit_sparse") + ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true", + "inverted_index_storage_format"="SNII")""" + // Row 1 is a dense3 false positive, but not a sparse4 candidate. Both indexes must + // preserve row 0 and the scalar LIKE/REGEXP recheck must reject row 1. + sql """INSERT INTO test_gram_metadata_inherit VALUES + (0, 'abcdefgh', 'abcdefgh', 'red fox'), + (1, 'abc!bcd!cde!def!efg!fgh', 'abc!bcd!cde!def!efg!fgh', 'blue fox'), + (2, 'unrelated', 'unrelated', 'green dog')""" + sql "sync" + sql "SET enable_sql_cache=false" + // The condition cache must be off too: gram deliberately keeps its LIKE / REGEXP expression + // in _common_expr_ctxs_push_down for the row-level recheck, so the segment iterator never + // zeroes the condition cache digest, and that digest ignores enable_inverted_index_query. + // Left on, the index-on pass fills the per-segment granule cache and the index-off pass hits + // it, so both passes would read one and the same filter result and this parity check would + // degenerate into a tautology. + sql "SET enable_condition_cache=false" + + def checkPatterns = { String phase -> + [false, true].each { useIndex -> + sql "SET enable_inverted_index_query=${useIndex}" + ['dense', 'sparse'].each { column -> + "order_qt_${phase}_${column}_${useIndex}_like"("""SELECT id + FROM test_gram_metadata_inherit WHERE ${column} LIKE '%abcdefgh%'""") + "order_qt_${phase}_${column}_${useIndex}_regexp"("""SELECT id + FROM test_gram_metadata_inherit WHERE ${column} REGEXP 'abcdefgh'""") + } + } + + // A correct full-scan fallback must not hide lost gram metadata after inheritance. + // The framework waits for the completed profile instead of relying on a fixed sleep. + sql "SET enable_profile=true" + sql "SET profile_level=2" + try { + ['dense', 'sparse'].each { column -> + [like: "LIKE '%abcdefgh%'", regexp: "REGEXP 'abcdefgh'"].each { kind, predicate -> + def profileId = "gram_inherit_${phase}_${column}_${kind}_${System.nanoTime()}" + sql """/* ${profileId} */ SELECT id FROM test_gram_metadata_inherit + WHERE ${column} ${predicate} ORDER BY id""" + def profile = new ProfileAction(context).getProfileBySql( + profileId, ["RowsGramIndexFiltered"]) + def filtered = Pattern.compile("RowsGramIndexFiltered:\\s*(\\d+)") + .matcher(profile) + assertTrue(filtered.find(), "RowsGramIndexFiltered missing from ${profileId}") + assertTrue(Long.parseLong(filtered.group(1)) > 0, + "${profileId} must still use the gram index") + // Evidence that "SET enable_condition_cache=false" above took effect: a + // non-zero ConditionCacheHit would mean the index-on and index-off passes + // share one cached per-granule filter result, so the comparison above would + // no longer be able to expose a row the gram index dropped. + def cacheHit = Pattern.compile("ConditionCacheHit:[^\\n]*[1-9][^\\n]*") + .matcher(profile) + assertTrue(!cacheHit.find(), + "${profileId} hit the condition cache, so index on/off no longer " + + "compare two independent scans") + } + } + } finally { + sql "SET enable_profile=false" + } + } + + checkPatterns("before") + // Add only metadata here: old rowsets still lack idx_tag, and retain both gram indexes. + // The following named BUILD INDEX must copy those unchanged logical indexes into the + // replacement SNII container while it builds the new English dictionary from raw tag data. + sql "SET enable_add_index_for_new_data=true" + sql """CREATE INDEX idx_tag ON test_gram_metadata_inherit(tag) USING INVERTED + PROPERTIES ("parser"="english")""" + sql "BUILD INDEX idx_tag ON test_gram_metadata_inherit" + wait_for_last_build_index_finish("test_gram_metadata_inherit", 180_000) + // Require a real FINISHED job even if the shared wait helper reaches its timeout. + def jobs = sql_return_maparray("""SHOW BUILD INDEX + WHERE TableName='test_gram_metadata_inherit' ORDER BY JobId DESC LIMIT 1""") + if (jobs.isEmpty() || jobs[0].State != "FINISHED") { + throw new IllegalStateException("gram inheritance BUILD INDEX did not finish: ${jobs}") + } + sql "sync" + checkPatterns("after") + + // Disallow MATCH's scalar fallback: success on the old rows proves the newly built index + // is usable as well as the inherited gram dictionaries. + order_qt_new_index """SELECT /*+ SET_VAR(enable_match_without_inverted_index=false) */ id + FROM test_gram_metadata_inherit WHERE tag MATCH 'fox'""" + order_qt_inherited_and_new """SELECT /*+ SET_VAR(enable_match_without_inverted_index=false) */ id + FROM test_gram_metadata_inherit WHERE dense LIKE '%abcdefgh%' AND tag MATCH 'fox'""" +} diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy new file mode 100644 index 00000000000000..3cba5eb313bb0d --- /dev/null +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_gram_pattern_recall", "p0") { + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_recall_invalid_tokenizer" + test { + sql """CREATE INVERTED INDEX TOKENIZER gram_recall_invalid_tokenizer + PROPERTIES ("type"="ngram", "mode"="sparse", "density"="0.25f")""" + exception "density must be a decimal number" + } + def sqlLiteral = { String value -> + "'" + value.replace('\\', '\\\\').replace("'", "\\'") + .replace('\u0000', '\\0') + "'" + } + def waitAnalyzerInstalled = { String name -> + def deadline = System.currentTimeMillis() + 180_000 + Exception lastNotFound = null + while (System.currentTimeMillis() < deadline) { + try { + sql """SELECT TOKENIZE('probe', '"analyzer"="${name}"')""" + return + } catch (Exception e) { + if (!e.message.contains("Policy not found")) { + throw e; + } + lastNotFound = e + sleep(1000) + } + } + throw new IllegalStateException("analyzer ${name} was not installed on BE", lastNotFound); + } + + sql "DROP TABLE IF EXISTS test_gram_pattern_recall" + def schemes = [dense: ["dense", false], sparse: ["sparse", false], + dense_lc: ["dense", true], sparse_lc: ["sparse", true]] + schemes.each { name, properties -> + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_recall_${name}" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_recall_${name}_tok" + sql """CREATE INVERTED INDEX TOKENIZER gram_recall_${name}_tok PROPERTIES ( + "type"="ngram", "mode"="${properties[0]}", "min_gram"="3", + "lower_case"="${properties[1]}")""" + sql """CREATE INVERTED INDEX ANALYZER gram_recall_${name} + PROPERTIES ("tokenizer"="gram_recall_${name}_tok")""" + waitAnalyzerInstalled("gram_recall_${name}") + } + sql """CREATE TABLE test_gram_pattern_recall ( + id INT, + dense VARCHAR(128), sparse VARCHAR(128), dense_lc VARCHAR(128), sparse_lc VARCHAR(128), + INDEX idx_dense (dense) USING INVERTED PROPERTIES ("analyzer"="gram_recall_dense"), + INDEX idx_sparse (sparse) USING INVERTED PROPERTIES ("analyzer"="gram_recall_sparse"), + INDEX idx_dense_lc (dense_lc) USING INVERTED PROPERTIES ("analyzer"="gram_recall_dense_lc"), + INDEX idx_sparse_lc (sparse_lc) USING INVERTED PROPERTIES ("analyzer"="gram_recall_sparse_lc") + ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true", + "inverted_index_storage_format"="SNII")""" + + def rows = ['é', 'É', 'ask', 'aſk', 'asK', 'abÉtimeout', + 'ab\u0007cdtimeout', 'ab\fcdtimeout', 'ab\u000bcdtimeout', + 'ab\ncdtimeout', 'ab\rcdtimeout', 'abacdtimeout', 'timeout', 'timeoutx', + 'ab cdtimeout', 'ab\tcdtimeout', 'abcdeftimeout', 'abcDEFtimeout', + 'ABCdeftimeout', 'ab', 'abcccdef', 'abcctimeout', 'abcabc', + 'abéétimeout', 'atimeout', 'aaatimeout', 'prefix', 'prefixsuffix', + 'prefix\u0000suffix', 'unrelated', '', null, '%timeout%', 'a.*timeout', + 'prefix1', 'prefix1\u0000timeout'] + // Separate inserts retain multiple rowsets and exercise per-segment candidate/null bitmaps. + rows.withIndex().collate(17).each { batch -> + def values = batch.collect { value, id -> + def literal = value == null ? 'NULL' : sqlLiteral(value) + "(${id}, ${literal}, ${literal}, ${literal}, ${literal})" + }.join(',') + sql "INSERT INTO test_gram_pattern_recall VALUES ${values}" + sql "sync" + } + + def patterns = ['(?i)é', '(?i)[é]', '(?i)ask', '(?i)a[s]k', '(?i)as[k]', + 'ab(?i)é(?-i:timeout)', 'ab(?i)[é](?-i:timeout)', + 'ab\\acdtimeout', 'ab[\\a]cdtimeout', 'ab\\fcdtimeout', + 'ab[\\f]cdtimeout', 'ab\\vcdtimeout', 'ab[\\v]cdtimeout', + 'ab\\141cdtimeout', 'ab[\\141]cdtimeout', 'timeout\\Z', + 'ab\\hcdtimeout', '(?i)((?-i)abc)DEF(?-i:timeout)', + '(?i)(?:(?-i)abc)DEF(?-i:timeout)', + '(?i)(?P(?-i)abc)DEF(?-i:timeout)', + '\\Qabc\\E*', '\\Qabc\\E+def', '\\Qabc\\E{2}timeout', + '(\\Qabc\\E)+', '\\Qabé\\E{2}timeout', 'a\\Q\\E*timeout', + 'prefix\u0000suffix', 'prefix\\x00suffix', 'prefix[0-9]\u0000timeout'] + sql "SET enable_sql_cache=false" + // The condition cache must be off too: gram deliberately keeps its LIKE / REGEXP expression + // in _common_expr_ctxs_push_down for the row-level recheck, so the segment iterator never + // zeroes the condition cache digest, and that digest ignores enable_inverted_index_query. + // Left on, the index-on pass fills the per-segment granule cache and the index-off pass hits + // it, so both passes would read one and the same filter result and this parity check would + // degenerate into a tautology. + sql "SET enable_condition_cache=false" + [false, true].each { useIndex -> + sql "SET enable_inverted_index_query=${useIndex}" + schemes.each { column, unused -> + patterns.eachWithIndex { pattern, i -> + "order_qt_${column}_${useIndex}_${i}"("""SELECT id + FROM test_gram_pattern_recall WHERE ${column} REGEXP ${sqlLiteral(pattern)}""") + } + ['%prefix\u0000%timeout%', 'prefix_\u0000%timeout%', 'prefix\u0000suffix'] + .eachWithIndex { pattern, i -> + "order_qt_${column}_${useIndex}_nul_like_${i}"("""SELECT id + FROM test_gram_pattern_recall WHERE ${column} LIKE ${sqlLiteral(pattern)}""") + } + "order_qt_${column}_${useIndex}_reverse_like"("""SELECT id + FROM test_gram_pattern_recall WHERE 'abctimeout' LIKE ${column}""") + "order_qt_${column}_${useIndex}_reverse_regexp"("""SELECT id + FROM test_gram_pattern_recall WHERE 'abctimeout' REGEXP ${column}""") + "order_qt_${column}_${useIndex}_null"("""SELECT id + FROM test_gram_pattern_recall WHERE ${column} IS NULL""") + "order_qt_${column}_${useIndex}_compound"("""SELECT id + FROM test_gram_pattern_recall WHERE ${column} LIKE '%timeout%' + AND (${column} REGEXP 'abc' OR ${column} IS NULL)""") + } + } + sql "SET enable_inverted_index_query=true" +} diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy new file mode 100644 index 00000000000000..04c5b106151913 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_gram_policy_recovery", "p0") { + sql "SET enable_sql_cache=false" + // The condition cache must be off too: gram deliberately keeps its LIKE / REGEXP expression + // in _common_expr_ctxs_push_down for the row-level recheck, so the segment iterator never + // zeroes the condition cache digest, and that digest ignores enable_inverted_index_query. + // Left on, the index-on pass fills the per-segment granule cache and the index-off pass hits + // it, so both passes would read one and the same filter result and this parity check would + // degenerate into a tautology. + sql "SET enable_condition_cache=false" + + def waitAnalyzerInstalled = { int expectedTokenCount -> + def deadline = System.currentTimeMillis() + 180_000 + Exception lastNotFound = null + while (System.currentTimeMillis() < deadline) { + try { + def result = sql """SELECT JSON_LENGTH(TOKENIZE('abcdef', + '"analyzer"="gram_recovery_analyzer"'))""" + // The replacement has the same name: successful lookup alone could still + // observe the old policy before BE receives the next heartbeat update. + if (result[0][0].toString() == expectedTokenCount.toString()) { + return + } + } catch (Exception e) { + if (!e.message.contains("Policy not found")) { + throw e + } + lastNotFound = e + } + sleep(1000) + } + throw new IllegalStateException( + "gram_recovery_analyzer did not produce ${expectedTokenCount} tokens on BE", + lastNotFound) + } + + sql "DROP TABLE IF EXISTS test_gram_policy_recovery" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_recovery_analyzer" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_recovery_tokenizer" + sql """CREATE INVERTED INDEX TOKENIZER gram_recovery_tokenizer PROPERTIES ( + "type"="ngram", "mode"="dense", "min_gram"="3")""" + sql """CREATE INVERTED INDEX ANALYZER gram_recovery_analyzer + PROPERTIES ("tokenizer"="gram_recovery_tokenizer")""" + waitAnalyzerInstalled(4) + qt_dense3_tokens """SELECT TOKENIZE('abcdef', '"analyzer"="gram_recovery_analyzer"')""" + + sql """CREATE TABLE test_gram_policy_recovery ( + id INT, + msg VARCHAR(128), + INDEX idx_msg (msg) USING INVERTED PROPERTIES ("analyzer"="gram_recovery_analyzer") + ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true", + "inverted_index_storage_format"="SNII")""" + sql """INSERT INTO test_gram_policy_recovery VALUES + (1, 'abcdef'), (2, 'old abcdef row'), (3, 'unrelated'), (4, NULL)""" + sql "sync" + + // Do not FORCE: the physical dense3 segment must survive in the recycle bin. + sql "DROP TABLE test_gram_policy_recovery" + sql "DROP INVERTED INDEX ANALYZER gram_recovery_analyzer" + sql "DROP INVERTED INDEX TOKENIZER gram_recovery_tokenizer" + sql """CREATE INVERTED INDEX TOKENIZER gram_recovery_tokenizer PROPERTIES ( + "type"="ngram", "mode"="dense", "min_gram"="4")""" + sql """CREATE INVERTED INDEX ANALYZER gram_recovery_analyzer + PROPERTIES ("tokenizer"="gram_recovery_tokenizer")""" + waitAnalyzerInstalled(3) + qt_dense4_tokens """SELECT TOKENIZE('abcdef', '"analyzer"="gram_recovery_analyzer"')""" + sql "RECOVER TABLE test_gram_policy_recovery" + + [false, true].each { useIndex -> + sql "SET enable_inverted_index_query=${useIndex}" + "order_qt_recovered_${useIndex}_like"("""SELECT id FROM test_gram_policy_recovery + WHERE msg LIKE '%abcdef%'""") + "order_qt_recovered_${useIndex}_regexp"("""SELECT id FROM test_gram_policy_recovery + WHERE msg REGEXP 'abc.*def'""") + } + + // This writer sees dense4, while the recovered rowset retains its dense3 dictionary. + // Disable automatic compaction above so both physical schemes remain present together. + sql """INSERT INTO test_gram_policy_recovery VALUES + (5, 'new abcdef row'), (6, 'abc'), (7, 'another unrelated row'), (8, NULL)""" + sql "sync" + [false, true].each { useIndex -> + sql "SET enable_inverted_index_query=${useIndex}" + "order_qt_mixed_${useIndex}_like"("""SELECT id FROM test_gram_policy_recovery + WHERE msg LIKE '%abcdef%'""") + "order_qt_mixed_${useIndex}_regexp"("""SELECT id FROM test_gram_policy_recovery + WHERE msg REGEXP 'abc.*def'""") + // Three-character literals can prune dense3 but must not eliminate dense4 rows. + "order_qt_mixed_${useIndex}_short_like"("""SELECT id FROM test_gram_policy_recovery + WHERE msg LIKE '%abc%'""") + "order_qt_mixed_${useIndex}_short_regexp"("""SELECT id FROM test_gram_policy_recovery + WHERE msg REGEXP 'abc'""") + } + sql "SET enable_inverted_index_query=true" +} diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy new file mode 100644 index 00000000000000..6beca82b2b457f --- /dev/null +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy @@ -0,0 +1,373 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.regex.Pattern + +import org.apache.doris.regression.action.ProfileAction + +// A gram (sparse / dense ngram) index accelerates LIKE / REGEXP by "an approximate candidate +// superset plus expression re-verification": the index only narrows the candidate rows, and the +// final match is still decided row by row by the LIKE / REGEXP expression. So the core assertion +// of this suite is a **semantic comparison** -- the same query must return exactly the same set +// of rows with enable_inverted_index_query=true and with false. Any difference means the index +// pruned away a genuinely matching row (a false negative), which is a functional defect. +// +// Two safety nets: +// 1) the .out golden: each mode emits its own idx_true_* / idx_false_* sections, which must be +// byte-identical when compared by hand or by script; +// 2) programmatic assertions: runParityCheck compares the sorted id lists of the same queries +// in the two modes directly. +suite("test_gram_regexp_like", "p0") { + def tbl = "t_gram_regexp_like" + // Policy names are globally unique: tokenizers and analyzers share one namespace, so the two + // names must differ + def sparseTok = "gram_rl_sparse_tok" + def sparseAna = "gram_rl_sparse" + def denseLcTok = "gram_rl_dense_lc_tok" + def denseLcAna = "gram_rl_dense_lc" + + // An analyzer reaches BE asynchronously over the heartbeat, so before creating the table we + // must confirm BE has loaded it, otherwise the write side gets no gram scheme + def waitAnalyzerInstalled = { String name -> + def deadline = System.currentTimeMillis() + 180_000 + Exception lastNotFound = null + while (System.currentTimeMillis() < deadline) { + try { + sql """SELECT TOKENIZE('probe', '\"analyzer\"=\"${name}\"')""" + return + } catch (Exception e) { + if (!e.message.contains("Policy not found")) { + throw e; + } + lastNotFound = e + sleep(1000) + } + } + throw new IllegalStateException("analyzer ${name} was not installed on BE", lastNotFound); + } + + // Turn the groovy-side "raw regex/LIKE text" into the contents of a SQL string literal: + // Doris, like MySQL, eats one more layer of backslashes inside a string literal, so \. has to + // be written as \\. + def sqlEsc = { String s -> s.replace('\\', '\\\\').replace("'", "\\'") } + + sql "DROP TABLE IF EXISTS ${tbl}" + try_sql "DROP INVERTED INDEX ANALYZER IF EXISTS ${sparseAna}" + try_sql "DROP INVERTED INDEX ANALYZER IF EXISTS ${denseLcAna}" + try_sql "DROP INVERTED INDEX TOKENIZER IF EXISTS ${sparseTok}" + try_sql "DROP INVERTED INDEX TOKENIZER IF EXISTS ${denseLcTok}" + + // Sparse gram: density=0.25, min_gram..max_gram = 3..16 + sql """ + CREATE INVERTED INDEX TOKENIZER IF NOT EXISTS ${sparseTok} + PROPERTIES ( + "type" = "ngram", + "mode" = "sparse", + "min_gram" = "3", + "max_gram" = "16", + "density" = "0.25" + ) + """ + // A gram-family analyzer must be a "bare tokenizer" and may carry no token_filter (FE checks) + sql """ + CREATE INVERTED INDEX ANALYZER IF NOT EXISTS ${sparseAna} + PROPERTIES ("tokenizer" = "${sparseTok}") + """ + // Dense gram + lower_case: covers case-insensitive candidate recall + // ('code = unavailable' vs 'CODE = UNAVAILABLE') + sql """ + CREATE INVERTED INDEX TOKENIZER IF NOT EXISTS ${denseLcTok} + PROPERTIES ( + "type" = "ngram", + "mode" = "dense", + "min_gram" = "3", + "lower_case" = "true" + ) + """ + sql """ + CREATE INVERTED INDEX ANALYZER IF NOT EXISTS ${denseLcAna} + PROPERTIES ("tokenizer" = "${denseLcTok}") + """ + waitAnalyzerInstalled(sparseAna) + waitAnalyzerInstalled(denseLcAna) + + // Three INVERTED indexes on the same column: sparse gram / dense gram (lower_case) / english + // tokenized index. The first two verify that "whichever gram index the optimizer picks, the + // semantics stay the same", and the third that gram coexists with a language tokenizer index. + // A gram index is forced to docs-only and FE defaults support_phrase to false; the storage + // format must be SNII. + sql """ + CREATE TABLE ${tbl} ( + id INT, + msg VARCHAR(512), + INDEX idx_msg_gram (msg) USING INVERTED PROPERTIES ("analyzer" = "${sparseAna}"), + INDEX idx_msg_lc (msg) USING INVERTED PROPERTIES ("analyzer" = "${denseLcAna}"), + INDEX idx_msg_en (msg) USING INVERTED PROPERTIES ("parser" = "english") + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "inverted_index_storage_format" = "SNII" + ) + """ + + // The first rowset / segment + sql """INSERT INTO ${tbl} VALUES + (1, 'rpc error: code = Unavailable desc = error reading from server'), + (2, 'user_id="eacb47f6-967d-11f0-b88d-8eb93cba8bdb" user_currency="USD"'), + (3, 'Convert conversion successful'), + (4, '手机微博 POST 10.68.3.18:8080 error'), + (5, NULL), + (6, ''), + (7, 'ab'), + (8, 'GET /images/x.gif HTTP/1.0'), + (9, 'CODE = UNAVAILABLE'), + (10, 'context deadline exceeded'), + (11, 'failed to charge card: rpc error'), + (12, 'timeout after error error error')""" + sql "sync" + // The second rowset / segment: checks that the candidate bitmap is still solved correctly per + // segment when there are several segments + sql """INSERT INTO ${tbl} VALUES + (13, 'rpc error: code = Internal desc = boom'), + (14, '微博手机'), + (15, 'abc'), + (16, 'Sending Quote: 12.5'), + (17, 'progress 100% done'), + (18, ' '), + (19, 'MiXeD CaSe UnAvAiLaBlE')""" + sql "sync" + + // Coverage matrix: long literal / wildcard / alternation / optional group / character class / + // no literal / anchors / escaped dot / case ((?i) and the lower_case index) / CJK / a literal + // shorter than min_gram / the empty pattern + def regexps = [ + 'rpc error: code = Unavailable', + 'error.*timeout', + 'code = (Unavailable|Internal)', + 'user_id="[0-9a-f]{8}-', + 'conn(ection)? re(set|fused)', + 'GET|POST', + '[0-9]{3}-[0-9]{4}', + '(?i)unavailable', + '手机微博', + '微博', + '^abc$', + 'a.*b', + 'Sending Quote: [0-9]+\\.[0-9]+', + 'failed to (convert|charge)', + '.*', + '[0-9]+', + 'code = unavailable', + 'error', + '\\.gif', + '^rpc', + 'exceeded$', + 'ab', + 'x', + '', + '用户|微博', + 'HTTP/1\\.[0-9]', + ] + + // LIKE coverage: the % / _ wildcards, the empty string, an exact match, multi-segment + // wildcards, CJK, and case + def likes = [ + '%rpc error%', + '%Unavail%', + '%手机%', + 'ab%', + '%', + '%x.gif%', + '%code = _navailable%', + '', + 'abc', + '%error%error%', + '_bc', + '%微博%', + '%CODE = UNAVAILABLE%', + ] + + // Compound predicates: an OR of REGEXPs, a REGEXP inside NOT (a AND b), REGEXP mixed with + // LIKE, and a LIKE with a custom ESCAPE (handled conservatively on the BE side: it is not + // pushed down, but it must still be correct) + def compounds = [ + "msg REGEXP 'rpc' OR msg REGEXP '微博'", + "NOT (msg REGEXP 'rpc' AND id > 5)", + "msg REGEXP 'error' AND id > 3", + "msg LIKE '%error%' AND msg REGEXP 'rpc'", + "NOT (msg REGEXP 'error') OR msg REGEXP 'abc'", + "NOT (msg LIKE '%rpc%' AND msg REGEXP 'error')", + "msg REGEXP 'code' AND msg NOT LIKE '%Internal%'", + "msg LIKE '%100!%%' ESCAPE '!'", + "msg LIKE '%100!% do%' ESCAPE '!'", + ] + + // Build the [tag, sql] list; the tag does not contain the raw pattern, which keeps the + // section names in .out stable and reproducible + def buildQueries = { -> + def qs = [] + regexps.eachWithIndex { p, i -> + def e = sqlEsc(p) + qs << ["regexp_${i}".toString(), + "SELECT id FROM ${tbl} WHERE msg REGEXP '${e}'".toString()] + qs << ["rlike_${i}".toString(), + "SELECT id FROM ${tbl} WHERE msg RLIKE '${e}'".toString()] + qs << ["notregexp_${i}".toString(), + "SELECT id FROM ${tbl} WHERE msg NOT REGEXP '${e}'".toString()] + qs << ["regexp_and_${i}".toString(), + "SELECT id FROM ${tbl} WHERE msg REGEXP '${e}' AND id > 3".toString()] + } + likes.eachWithIndex { p, i -> + def e = sqlEsc(p) + qs << ["like_${i}".toString(), + "SELECT id FROM ${tbl} WHERE msg LIKE '${e}'".toString()] + qs << ["notlike_${i}".toString(), + "SELECT id FROM ${tbl} WHERE msg NOT LIKE '${e}'".toString()] + } + compounds.eachWithIndex { c, i -> + qs << ["compound_${i}".toString(), + "SELECT id FROM ${tbl} WHERE ${c}".toString()] + } + return qs + } + + def queries = buildQueries() + log.info("gram parity matrix: ${queries.size()} queries x 2 modes".toString()) + + // Turn the SQL cache off, so the two modes cannot reuse one cached result and blur the + // comparison + sql "SET enable_sql_cache=false" + // The condition cache must be off too: gram deliberately keeps its LIKE / REGEXP expression + // in _common_expr_ctxs_push_down for the row-level recheck, so the segment iterator never + // zeroes the condition cache digest, and that digest ignores enable_inverted_index_query. + // Left on, the index-on pass fills the per-segment granule cache and the index-off pass hits + // it, so both passes would read one and the same filter result and this parity check would + // degenerate into a tautology. + sql "SET enable_condition_cache=false" + + // Produce the .out golden: run the same batch of queries once with the index on and once off + def runAll = { boolean useIndex -> + sql "SET enable_inverted_index_query=${useIndex}" + queries.each { entry -> + def tag = "${entry[0]}_idx_${useIndex}".toString() + "order_qt_${tag}"(entry[1]) + } + "order_qt_count_idx_${useIndex}"("SELECT count(*) FROM ${tbl} WHERE msg REGEXP 'error'".toString()) + } + runAll(true) + runAll(false) + + // Programmatic safety net: instead of eyeballing .out, compare the sorted id lists of the two + // modes query by query + def idsOf = { String q -> + return sql(q).collect { it[0] as Integer }.sort() + } + def runParityCheck = { String phase -> + def mismatches = [] + queries.each { entry -> + sql "SET enable_inverted_index_query=true" + def withIdx = idsOf(entry[1]) + sql "SET enable_inverted_index_query=false" + def noIdx = idsOf(entry[1]) + if (withIdx != noIdx) { + mismatches << "[${phase}][${entry[0]}] ${entry[1]}\n idx_on =${withIdx}\n idx_off=${noIdx}".toString() + } + } + if (!mismatches.isEmpty()) { + throw new AssertionError( + "gram index changed query semantics (${mismatches.size()} mismatches):\n" + + mismatches.join("\n") as Object) + } + log.info("gram parity check [${phase}] passed for ${queries.size()} queries".toString()) + } + runParityCheck("base") + + // The profile proves the gram index really took part in the pruning: RowsGramIndexFiltered > 0. + // A counter may be rendered as "18" or as "12.0K (12000)", and both forms must be parseable. + def parseProfileCounter = { String profileString, String name -> + def exact = Pattern.compile(Pattern.quote(name) + ":\\s*[^\\(\\n]*\\((\\d+)\\)").matcher(profileString) + if (exact.find()) { + return Long.parseLong(exact.group(1)) + } + def plain = Pattern.compile(Pattern.quote(name) + ":\\s*(\\d+)").matcher(profileString) + assertTrue(plain.find(), "${name} is not parseable from profile") + return Long.parseLong(plain.group(1)) + } + // Evidence that "SET enable_condition_cache=false" above really took effect: FE then sends no + // condition cache digest, so the scanners render "ConditionCacheHit: 0" (or, in the merged + // profile, "sum 0, avg 0, max 0, min 0"). A build that does not export the counter at all is + // equally fine -- what must never appear is a non-zero hit, because that would mean the + // index-on and index-off passes can share one cached per-granule filter result and every + // parity comparison in this suite would silently become a tautology. + def assertConditionCacheUnused = { String label, String profileString -> + // Any non-zero digit inside the rendered value means the cache served this scan. + def matcher = Pattern.compile("ConditionCacheHit:[^\\n]*[1-9][^\\n]*").matcher(profileString) + def hits = [] + while (matcher.find()) { + hits << matcher.group().trim() + } + assertTrue(hits.isEmpty(), + "[${label}] the condition cache served this scan (${hits}); index on and index " + + "off would then compare one shared cached result, not two real scans") + } + def checkGramPruned = { String label, String profileString, Throwable exception -> + assertNull(exception) + assertTrue(profileString.contains("RowsGramIndexFiltered"), + "RowsGramIndexFiltered is missing from profile") + assertConditionCacheUnused(label, profileString) + def filtered = parseProfileCounter(profileString, "RowsGramIndexFiltered") + def candidate = parseProfileCounter(profileString, "GramIndexCandidateRows") + log.info("[${label}] RowsGramIndexFiltered=${filtered}, GramIndexCandidateRows=${candidate}".toString()) + assertTrue(filtered > 0, "[${label}] RowsGramIndexFiltered must be positive, got ${filtered}") + } + + sql "SET enable_inverted_index_query=true" + sql "set enable_profile=true" + sql "set profile_level=2" + // The profile is reported asynchronously by FE, so a fixed sleep either wastes time or + // occasionally comes up empty on a slow machine. Use the framework's own bounded polling + // ProfileAction#getProfileBySql instead: wait at most 30 s (every 500 ms) until this SQL's + // profile reads "Profile Completion State: COMPLETE" and both gram counters have been + // rendered; a timeout fails the test. + def gramProfileCounters = ["RowsGramIndexFiltered", "GramIndexCandidateRows"] + def profileAction = new ProfileAction(context) + // REGEXP takes the gram acceleration + order_qt_profile_q "/* gram_regexp_profile */ SELECT id FROM ${tbl} WHERE msg REGEXP 'context deadline exceeded'" + checkGramPruned("regexp", + profileAction.getProfileBySql("gram_regexp_profile", gramProfileCounters), null) + // LIKE takes the gram acceleration too + order_qt_profile_q_like "/* gram_like_profile */ SELECT id FROM ${tbl} WHERE msg LIKE '%Sending Quote%'" + checkGramPruned("like", + profileAction.getProfileBySql("gram_like_profile", gramProfileCounters), null) + sql "set enable_profile=false" + + // Query again after a delete: the delete predicate combined with the gram candidate bitmap + // must still agree with the no-index path + sql "SET enable_inverted_index_query=true" + sql "DELETE FROM ${tbl} WHERE id = 1" + sql "sync" + order_qt_after_delete_idx_true "SELECT id FROM ${tbl} WHERE msg REGEXP 'rpc error'" + sql "SET enable_inverted_index_query=false" + order_qt_after_delete_idx_false "SELECT id FROM ${tbl} WHERE msg REGEXP 'rpc error'" + // Compare the whole matrix once more after the delete + runParityCheck("after_delete") + + sql "SET enable_inverted_index_query=true" +} diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_schema_change.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_schema_change.groovy new file mode 100644 index 00000000000000..d52ff3bb484393 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_schema_change.groovy @@ -0,0 +1,270 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction + +import java.util.regex.Pattern + +// A gram index has to survive every schema change a table can go through, and until now +// nothing checked that it does. The changes reach the index in three different ways: +// +// - a light schema change (ADD/DROP COLUMN, a wider VARCHAR) touches no data, so the +// existing rowsets and their index files are carried over as they are; +// - a change that rewrites the data (a wider value or key type, a column reorder, VARCHAR +// to STRING on the indexed column itself) writes new segments, and each of them builds a +// fresh gram index from the converted column: it re-solves the density from its own rows, +// recomputes the df statistics and the high-df digest, and decides stop-gram anew, so the +// rebuilt index is not a copy of its input and cannot be assumed to behave like it; +// - a load that lands after a change writes under the new schema. +// +// What has to hold across all of them is the same: LIKE/REGEXP through the index return +// exactly what a scan returns, and the index is still there and still prunes rows afterwards +// rather than having been silently dropped or left unreadable. The profile proves the second +// half -- RowsGramIndexFiltered > 0 is only possible if the index was read and used. +suite("test_gram_schema_change", "p0") { + def waitAnalyzerInstalled = { String name -> + def deadline = System.currentTimeMillis() + 180_000 + Exception lastNotFound = null + while (System.currentTimeMillis() < deadline) { + try { + sql """SELECT TOKENIZE('probe', '"analyzer"="${name}"')""" + return + } catch (Exception e) { + if (!e.message.contains("Policy not found")) { + throw e + } + lastNotFound = e + sleep(1000) + } + } + throw new IllegalStateException("analyzer ${name} was not installed on BE", lastNotFound) + } + + def tableName = "test_gram_schema_change" + + // The table goes first: a policy still referenced by a table left behind by an earlier run + // cannot be dropped. + sql "DROP TABLE IF EXISTS ${tableName}" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_sc_ana" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_sc_tok" + sql """CREATE INVERTED INDEX TOKENIZER gram_sc_tok PROPERTIES ( + "type"="ngram", "mode"="sparse", "min_gram"="3", "max_gram"="8", "density"="0.5")""" + sql """CREATE INVERTED INDEX ANALYZER gram_sc_ana + PROPERTIES ("tokenizer"="gram_sc_tok")""" + waitAnalyzerInstalled("gram_sc_ana") + + // Two key columns, so that a key can be widened without touching the distribution column + // (FE refuses to modify that one); a value column to widen; and the indexed column as a + // bounded VARCHAR, so that both the length and the type of the indexed column itself can + // change later on. + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `k2` int NULL, + `v` int NULL, + `msg` varchar(256) NULL, + INDEX idx_msg (`msg`) USING INVERTED + PROPERTIES('analyzer'='gram_sc_ana', 'support_phrase'='false') + ) ENGINE=OLAP DUPLICATE KEY(`id`, `k2`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES('replication_num'='1', 'inverted_index_storage_format'='SNII', + 'disable_auto_compaction'='true')""" + + sql "SET enable_sql_cache=false" + sql "SET enable_condition_cache=false" + + // Row content spans the selectivity range: a marker on every row, one on a seventh of them, + // and a long needle on four rows out of the 5,600 the table ends up with. The needle is long + // enough for a sparse scheme to cut at least one gram out of it whatever density the segment + // settles on, and rare enough to keep its posting lists however the rows are laid out: a + // gram present in more than 0.15% of a segment's rows loses its posting list to stop-gram + // (df 6 in a 5,600-row segment) and then prunes nothing, which would read here exactly like + // an index the schema change had dropped. Auto compaction is off for the same reason -- the + // segment layout must change only when a schema change rewrites the data. Columns are named + // in the INSERT so that the same loader works before and after a column is added. + def perBatch = 800 + def loadBatch = { int b -> + def values = [] + for (int i = 0; i < perBatch; i++) { + def id = b * perBatch + i + def needle = (id % 1400 == 0) ? " unique_needle_in_haystack_${id}" : "" + def mid = (id % 7 == 0) ? " midfreq_token" : "" + values.add("(${id}, ${id % 97}, ${id}, 'shared_prefix_common_text filler_${id}${mid}${needle}')") + } + sql "INSERT INTO ${tableName} (id, k2, v, msg) VALUES ${values.join(',')}" + } + for (int b = 0; b < 5; b++) { + loadBatch(b) + } + + def patterns = [ + "like_needle" : "SELECT COUNT(*) FROM ${tableName} WHERE msg LIKE '%unique_needle_in_haystack%'", + "like_mid" : "SELECT COUNT(*) FROM ${tableName} WHERE msg LIKE '%midfreq_token%'", + "like_common" : "SELECT COUNT(*) FROM ${tableName} WHERE msg LIKE '%shared_prefix_common%'", + "regexp_needle" : "SELECT COUNT(*) FROM ${tableName} WHERE msg REGEXP 'unique_needle_in_haystack_[0-9]+'", + "regexp_mid" : "SELECT COUNT(*) FROM ${tableName} WHERE msg REGEXP 'midfreq_[a-z]+'", + "regexp_absent" : "SELECT COUNT(*) FROM ${tableName} WHERE msg REGEXP 'no_such_token_anywhere'", + "regexp_alt" : "SELECT COUNT(*) FROM ${tableName} WHERE msg REGEXP 'unique_needle_in_haystack_(0|1400|2800)\\\\b'", + "regexp_rows" : "SELECT id, v FROM ${tableName} WHERE msg REGEXP 'unique_needle_in_haystack_[0-9]+' ORDER BY id", + ] + + def runAll = { -> + def out = [:] + patterns.each { name, stmt -> out[name] = sql(stmt) } + return out + } + + // Ground truth is taken fresh at every step, with the index taken out of the picture + // entirely, because loads between the steps change the answers. + def assertParity = { String label -> + sql "SET enable_inverted_index_query=false" + def scanned = runAll() + sql "SET enable_inverted_index_query=true" + def indexed = runAll() + scanned.each { name, value -> + assertEquals(value, indexed[name], "[${label}] index and scan disagree on ${name}") + } + assertTrue((scanned["like_needle"][0][0] as long) > 0, "[${label}] the needle rows are gone") + logger.info("[${label}] parity held for ${patterns.size()} queries; needle rows: ${scanned['regexp_rows']}") + } + + // A counter may be rendered as "18" or as "12.0K (12000)", and both forms must be parseable. + def parseProfileCounter = { String profileString, String name -> + def exact = Pattern.compile(Pattern.quote(name) + ":\\s*[^\\(\\n]*\\((\\d+)\\)").matcher(profileString) + if (exact.find()) { + return Long.parseLong(exact.group(1)) + } + def plain = Pattern.compile(Pattern.quote(name) + ":\\s*(\\d+)").matcher(profileString) + assertTrue(plain.find(), "${name} is not parseable from profile") + return Long.parseLong(plain.group(1)) + } + def gramProfileCounters = ["RowsGramIndexFiltered", "GramIndexCandidateRows"] + def profileAction = new ProfileAction(context) + // The profile is reported asynchronously by FE; ProfileAction#getProfileBySql polls until + // this SQL's profile is complete and both gram counters have been rendered. + def assertIndexPrunes = { String label -> + def tag = "gram_sc_profile_${label}" + sql "SET enable_inverted_index_query=true" + sql "SET enable_profile=true" + sql "SET profile_level=2" + sql "/* ${tag} */ SELECT COUNT(*) FROM ${tableName} WHERE msg REGEXP 'unique_needle_in_haystack_[0-9]+'" + def profileString = profileAction.getProfileBySql(tag, gramProfileCounters) + sql "SET enable_profile=false" + def filtered = parseProfileCounter(profileString, "RowsGramIndexFiltered") + def candidate = parseProfileCounter(profileString, "GramIndexCandidateRows") + logger.info("[${label}] RowsGramIndexFiltered=${filtered}, GramIndexCandidateRows=${candidate}".toString()) + assertTrue(filtered > 0, "[${label}] the gram index pruned nothing (RowsGramIndexFiltered=${filtered}): " + + "it was dropped by the schema change or is no longer readable") + } + + def latestAlterJob = { -> + def rows = sql_return_maparray """SHOW ALTER TABLE COLUMN WHERE TableName = '${tableName}' + ORDER BY CreateTime DESC LIMIT 1""" + return rows.isEmpty() ? null : rows[0] + } + // Waits for the job this statement creates, identified by not being the job that was the + // latest one before it; a light schema change finishes synchronously, so its job is already + // FINISHED on the first look. + def alterAndWait = { String label, String stmt -> + def previous = latestAlterJob() + def previousId = previous == null ? null : previous.JobId + sql stmt + def deadline = System.currentTimeMillis() + 600_000 + def job = null + while (true) { + job = latestAlterJob() + if (job != null && job.JobId != previousId) { + if (job.State == "FINISHED") { + break + } + if (job.State == "CANCELLED") { + throw new IllegalStateException("[${label}] schema change was cancelled: ${job.Msg}") + } + } + assertTrue(System.currentTimeMillis() < deadline, + "[${label}] schema change did not finish within 600 s, last job: ${job}") + sleep(1000) + } + logger.info("[${label}] finished: ${stmt}".toString()) + } + + def indexBytes = { -> + def deadline = System.currentTimeMillis() + 180_000 + long bytes = 0L + while (System.currentTimeMillis() < deadline) { + def rows = sql """SELECT INDEX_LENGTH FROM information_schema.tables + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '${tableName}'""" + bytes = rows.isEmpty() ? 0L : (rows[0][0] as long) + if (bytes > 0) { + break + } + sleep(2000) + } + return bytes + } + + assertParity("base") + assertIndexPrunes("base") + def bytesBefore = indexBytes() + logger.info("index bytes before any schema change: ${bytesBefore}") + assertTrue(bytesBefore > 0, "the gram index must occupy bytes before any schema change") + + // Light: a new column, then a load that lands under the new schema. + alterAndWait("add_column", "ALTER TABLE ${tableName} ADD COLUMN extra INT DEFAULT \"0\"") + assertParity("add_column") + assertIndexPrunes("add_column") + loadBatch(5) + assertParity("load_after_add_column") + assertIndexPrunes("load_after_add_column") + + // Rewrites: a wider value type, then a wider key type (the sorting path). + alterAndWait("widen_value", "ALTER TABLE ${tableName} MODIFY COLUMN v BIGINT") + assertParity("widen_value") + assertIndexPrunes("widen_value") + alterAndWait("widen_key", "ALTER TABLE ${tableName} MODIFY COLUMN k2 BIGINT KEY") + assertParity("widen_key") + assertIndexPrunes("widen_key") + + // The indexed column itself: a longer VARCHAR, then VARCHAR to STRING. + alterAndWait("widen_indexed_varchar", "ALTER TABLE ${tableName} MODIFY COLUMN msg VARCHAR(1024)") + assertParity("widen_indexed_varchar") + assertIndexPrunes("widen_indexed_varchar") + alterAndWait("indexed_varchar_to_string", "ALTER TABLE ${tableName} MODIFY COLUMN msg STRING") + assertParity("indexed_varchar_to_string") + assertIndexPrunes("indexed_varchar_to_string") + + // Light again: drop the added column; then a rewrite that only reorders the columns. + alterAndWait("drop_column", "ALTER TABLE ${tableName} DROP COLUMN extra") + assertParity("drop_column") + assertIndexPrunes("drop_column") + alterAndWait("reorder", "ALTER TABLE ${tableName} ORDER BY (id, k2, msg, v)") + assertParity("reorder") + assertIndexPrunes("reorder") + + // A load under the final schema, and the rebuilt index still occupies bytes. + loadBatch(6) + assertParity("load_after_all") + assertIndexPrunes("load_after_all") + def bytesAfter = indexBytes() + logger.info("index bytes after all schema changes: ${bytesAfter}") + assertTrue(bytesAfter > 0, "the gram index must still occupy bytes after the schema changes") + + sql "SET enable_inverted_index_query=true" + sql "DROP TABLE IF EXISTS ${tableName}" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_sc_ana" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_sc_tok" +} diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy new file mode 100644 index 00000000000000..f5afeb0e7a5c21 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// stop-gram end to end: an index built with the posting lists of very common grams dropped +// must still answer LIKE/REGEXP exactly, and must actually have dropped them. +// +// A dropped gram matches every document, so it can only ever widen the candidate set the +// index proposes, and the predicate is re-evaluated on those candidates. The observable +// contract is therefore equality: every pattern must return exactly what a full scan +// returns. The rows below are built so that the common grams really are common -- a shared +// prefix on every row -- while the patterns being searched for are rare, which is the shape +// where dropping matters. +// +// There is no switch to turn the feature off, so the control arm is the row floor instead: +// a segment under kHighDfDigestDivisor (2000) rows never drops a posting. The same 4000 rows +// are therefore loaded twice, once as a single segment (dropping active, threshold +// 4000 / 2000 * 3 = df 6) and once as five segments of 800 (dropping inactive, every +// posting kept). The two tables must agree on every answer, and the single-segment index +// must be much smaller -- that difference is the dropped postings, and it also pins that +// the floor holds. +suite("test_gram_stop_gram", "p0") { + def waitAnalyzerInstalled = { String name -> + def deadline = System.currentTimeMillis() + 180_000 + Exception lastNotFound = null + while (System.currentTimeMillis() < deadline) { + try { + sql """SELECT TOKENIZE('probe', '"analyzer"="${name}"')""" + return + } catch (Exception e) { + if (!e.message.contains("Policy not found")) { + throw e + } + lastNotFound = e + sleep(1000) + } + } + throw new IllegalStateException("analyzer ${name} was not installed on BE", lastNotFound) + } + + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort) + def set_be_config = { key, value -> + for (String backend_id : backendId_to_backendIP.keySet()) { + def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), + backendId_to_backendHttpPort.get(backend_id), key, value) + logger.info("update ${key}=${value}: code=${code}, out=${out}, err=${err}") + } + } + + // The tables go first: a policy still referenced by a table left behind by an earlier run + // cannot be dropped. + sql "DROP TABLE IF EXISTS test_gram_stop_gram_one" + sql "DROP TABLE IF EXISTS test_gram_stop_gram_five" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_stop_ana" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_stop_tok" + sql """CREATE INVERTED INDEX TOKENIZER gram_stop_tok PROPERTIES ( + "type"="ngram", "mode"="sparse", "min_gram"="3", "max_gram"="8", "density"="0.5")""" + sql """CREATE INVERTED INDEX ANALYZER gram_stop_ana + PROPERTIES ("tokenizer"="gram_stop_tok")""" + waitAnalyzerInstalled("gram_stop_ana") + + def rows = 4000 + def values = [] + for (int i = 0; i < rows; i++) { + // Every row shares "shared_prefix_", making its grams as common as a gram gets. Four + // rows carry a rare marker (df 4, under the single segment's threshold of 6, so its + // postings survive and the rare queries still filter); one row in seven carries a + // mid-frequency tail whose grams sit above the line and are dropped. + def tail = (i % 1000 == 0) ? "rare_marker_${i}" : "filler_${i}" + values.add("(${i}, 'shared_prefix_common_text ${tail} tail_${i % 7}')") + } + + def runAll = { String table, String label -> + def out = [:] + out["like_rare"] = sql "SELECT COUNT(*) FROM ${table} WHERE msg LIKE '%rare_marker_%'" + out["like_common"] = sql "SELECT COUNT(*) FROM ${table} WHERE msg LIKE '%shared_prefix_common%'" + out["regexp_rare"] = sql "SELECT COUNT(*) FROM ${table} WHERE msg REGEXP 'rare_marker_[0-9]+'" + out["regexp_common"] = sql "SELECT COUNT(*) FROM ${table} WHERE msg REGEXP 'shared_prefix_[a-z]+'" + out["regexp_mid"] = sql "SELECT COUNT(*) FROM ${table} WHERE msg REGEXP 'tail_3\$'" + out["regexp_alt"] = sql "SELECT COUNT(*) FROM ${table} WHERE msg REGEXP 'rare_marker_(0|1000|2000)\\\\b'" + out["regexp_absent"] = sql "SELECT COUNT(*) FROM ${table} WHERE msg REGEXP 'no_such_token_anywhere'" + out["like_tail"] = sql "SELECT COUNT(*) FROM ${table} WHERE msg LIKE '%tail_3'" + logger.info("${label}: ${out}") + return out + } + + def createTable = { String table -> + sql "DROP TABLE IF EXISTS ${table}" + sql """CREATE TABLE ${table} ( + `id` bigint NULL, + `msg` text NULL, + INDEX idx_msg (`msg`) USING INVERTED + PROPERTIES('analyzer'='gram_stop_ana', 'support_phrase'='false') + ) ENGINE=OLAP DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES('replication_num'='1', 'inverted_index_storage_format'='SNII', + 'disable_auto_compaction'='true')""" + } + + // information_schema reports a fresh table's sizes with a delay, and 0 there means "not + // yet" rather than "empty". + def indexBytes = { String table -> + def deadline = System.currentTimeMillis() + 180_000 + while (System.currentTimeMillis() < deadline) { + def r = sql """SELECT INDEX_LENGTH FROM information_schema.tables + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '${table}'""" + def bytes = r.isEmpty() ? 0L : (r[0][0] as long) + if (bytes > 0) { + return bytes + } + sleep(3000) + } + throw new IllegalStateException("index size of ${table} never reported") + } + + def oneSegment = "test_gram_stop_gram_one" + def fiveSegments = "test_gram_stop_gram_five" + try { + sql "SET enable_sql_cache=false" + sql "SET enable_condition_cache=false" + + // One batch keeps all 4000 rows in a single segment: above the row floor, dropping + // is in effect. + createTable(oneSegment) + sql "INSERT INTO ${oneSegment} VALUES ${values.join(',')}" + + // The same rows in five batches of 800: each segment is under the floor, so every + // posting list is kept. This is what the index looked like before dropping existed. + createTable(fiveSegments) + for (int b = 0; b < 5; b++) { + sql "INSERT INTO ${fiveSegments} VALUES ${values.subList(b * 800, (b + 1) * 800).join(',')}" + } + + // Ground truth, taken with the index out of the picture entirely. + sql "SET enable_inverted_index_query=false" + def scanned = runAll(oneSegment, "full scan") + sql "SET enable_inverted_index_query=true" + + def dropped = runAll(oneSegment, "one segment, postings dropped") + def kept = runAll(fiveSegments, "five segments, postings kept") + scanned.each { name, value -> + assertEquals(value[0][0], dropped[name][0][0], + "index and scan disagree on ${name} with postings dropped") + assertEquals(value[0][0], kept[name][0][0], + "index and scan disagree on ${name} with postings kept") + } + + // The postings really were dropped. Measured: the single-segment index is 0.36 of + // the five-segment one. Had nothing been dropped it would be about 0.9 -- the only + // difference left would be four fewer per-segment dictionaries -- so a factor of two + // separates the two outcomes with margin on both sides. + def droppedBytes = indexBytes(oneSegment) + def keptBytes = indexBytes(fiveSegments) + logger.info("index bytes: one segment ${droppedBytes}, five segments ${keptBytes}") + assertTrue(droppedBytes * 2 < keptBytes, + "the single-segment index (${droppedBytes} bytes) should be far smaller than " + + "the five-segment one (${keptBytes} bytes): its common postings were not dropped") + + // The gate could otherwise mask a broken dropped-gram path by giving up first, so + // check again with its fallback ratio disabled. The gate's primary budget comes from + // the segment itself and cannot be switched off -- that is the point of deriving it + // rather than configuring it -- but zeroing the ratio removes the one arm that a + // configuration could have been hiding behind. + set_be_config("gram_index_max_candidate_ratio_bp", "0") + def ratioOff = runAll(oneSegment, "postings dropped, fallback ratio off") + scanned.each { name, value -> + assertEquals(value[0][0], ratioOff[name][0][0], + "with the fallback ratio disabled the index changed the answer for ${name}") + } + } finally { + set_be_config("gram_index_max_candidate_ratio_bp", "15") + sql "DROP TABLE IF EXISTS ${oneSegment}" + sql "DROP TABLE IF EXISTS ${fiveSegments}" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_stop_ana" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_stop_tok" + } +} From 6cbfa9905f5e444d86fb98bbb06dfb719ddd1032 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 11 Sep 2026 09:49:24 +0800 Subject: [PATCH 07/36] [fix](be) skip analyzed MATCH queries on gram segments cut with a different scheme MATCH_* terms come out of the current analyzer. On a gram-family index they are grams, and they only mean what the segment's grams mean when both were cut by the same scheme. Two things make the schemes differ: adaptive density persists a rate solved from the segment's own rows, and a policy recreated with other parameters (a dense-3 index recovered under a dense-4 analyzer) changes what the current analyzer cuts. Until now only LIKE and REGEXP compiled against the segment's persisted scheme; a MATCH bitmap on such a segment could omit rows the scalar predicate returns, and the result cache, keyed by the raw query bytes, could hand that bitmap back without opening the segment at all. SniiIndexReader now resolves the scheme the current analyzer cuts with (the caller's provider, else the analyzer the index properties name) for every analyzed query type, compares it with the segment's persisted scheme once the logical reader is open, and returns INVERTED_INDEX_EVALUATE_SKIPPED on a mismatch so the predicate is evaluated without the index. Analyzed queries on a gram-family index never enter the result cache, whose key does not tell the two schemes apart. Tests: LikeGramBindingTest.MatchIsSkippedWhenTheCurrentSchemeDiffersFrom TheSegments (skip on a dense-4 analyzer over a dense-3 segment, answer through the segment's own scheme); test_gram_policy_recovery adds MATCH_ANY / MATCH_ALL parity with the index on and off on the recovered table and on the mixed dense-3 / dense-4 table. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01METuP3aVRn8dfCU62PivnF --- .../storage/index/snii/snii_index_reader.cpp | 61 ++++++++++++++++++- be/src/storage/index/snii/snii_index_reader.h | 6 ++ .../exprs/function/like_gram_binding_test.cpp | 46 ++++++++++++++ .../gram/test_gram_policy_recovery.out | 36 +++++++++++ .../gram/test_gram_policy_recovery.groovy | 13 ++++ 5 files changed, 161 insertions(+), 1 deletion(-) diff --git a/be/src/storage/index/snii/snii_index_reader.cpp b/be/src/storage/index/snii/snii_index_reader.cpp index 94f77dc1c5086d..1122a95090e206 100644 --- a/be/src/storage/index/snii/snii_index_reader.cpp +++ b/be/src/storage/index/snii/snii_index_reader.cpp @@ -42,7 +42,9 @@ #include "storage/index/index_reader_helper.h" #include "storage/index/inverted/analyzer/analyzer.h" #include "storage/index/inverted/common/single_flight.h" +#include "storage/index/inverted/gram/gram_family.h" #include "storage/index/inverted/gram/gram_query.h" +#include "storage/index/inverted/gram/gram_scheme.h" #include "storage/index/inverted/gram/regex_gram_compiler.h" #include "storage/index/inverted/inverted_index_cache.h" #include "storage/index/inverted/inverted_index_iterator.h" @@ -163,6 +165,23 @@ bool uses_phrase_frequency_scoring(InvertedIndexQueryType query_type, query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY); } +// Query types whose terms come out of the current analyzer. On a gram-family index those +// terms are grams, and they only mean what the segment's own grams mean when both were cut +// by the same scheme; a gram query compiles against the segment's scheme itself and a raw +// pattern query never analyzes, so neither is affected. +bool analyzes_query_terms(InvertedIndexQueryType query_type) { + switch (query_type) { + case InvertedIndexQueryType::MATCH_ANY_QUERY: + case InvertedIndexQueryType::MATCH_ALL_QUERY: + case InvertedIndexQueryType::MATCH_PHRASE_QUERY: + case InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY: + case InvertedIndexQueryType::MATCH_PHRASE_EDGE_QUERY: + return true; + default: + return false; + } +} + Status score_plain_term_candidates(const IndexQueryContextPtr& context, std::string_view column_name, const InvertedIndexQueryInfo& query_info, @@ -508,6 +527,27 @@ Status SniiIndexReader::new_iterator(std::unique_ptr* iterator) { return Status::OK(); } +// The scheme the current analyzer cuts query terms with: the provider the caller resolved, +// or, when none was handed down, the analyzer the index properties name -- the same fallback +// _parse_query_terms takes. A policy that no longer exists is reported, not swallowed: the +// analysis below would fail on it anyway. +Status SniiIndexReader::_current_gram_scheme( + const InvertedIndexAnalyzerCtx* analyzer_ctx, + std::optional* out) const { + if (analyzer_ctx != nullptr && analyzer_ctx->analyzer_provider != nullptr) { + *out = analyzer_ctx->analyzer_provider->gram_scheme(); + return Status::OK(); + } + try { + *out = segment_v2::gram::resolve_gram_scheme(_index_meta.properties(), + ExecEnv::GetInstance()->index_policy_mgr()); + } catch (const Exception& e) { + return Status::Error( + "SNII resolve analyzer failed: {}", e.what()); + } + return Status::OK(); +} + Status SniiIndexReader::_parse_query_terms(const IndexQueryContextPtr& context, std::string search_str, InvertedIndexQueryType query_type, @@ -687,11 +727,24 @@ Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::s if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { parse_phrase_slop(&plain_analysis_str, &query_info); } + // An analyzed query (MATCH_*) on a gram-family index is exact only when the query is cut + // by the very scheme that cut the segment. The segment's scheme is persisted -- a density + // solved from its own rows, or the scheme of a policy that has since been recreated -- + // while the analyzer here is the current one. Where they differ the scalar predicate and + // the index would disagree, so such a query is skipped once the segment is open, and it + // is never cached: the cache key does not tell the two schemes apart. + const bool analyzed_query = analyzes_query_terms(query_type) && + (analyzer_ctx == nullptr || analyzer_ctx->requires_analysis()); + std::optional current_gram_scheme; + if (analyzed_query) { + RETURN_IF_ERROR(_current_gram_scheme(analyzer_ctx, ¤t_gram_scheme)); + } // Result cache keys contain only (index file, column, query type, raw query bytes). Analysis // is determined by index properties and policies, which are immutable once referenced, so // sharing can be decided before opening the segment. Scoring queries depend on collection // statistics and use neither the result cache nor single-flight coalescing. - const bool allow_result_cache = !actual_similarity; + const bool allow_result_cache = + !actual_similarity && !(analyzed_query && current_gram_scheme.has_value()); const InvertedIndexRawQuerySemantic raw_semantic {.raw_query_bytes = search_str, .query_type = query_type, .slop = query_info.slop, @@ -714,6 +767,12 @@ Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::s const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, &logical_reader)); + if (analyzed_query && logical_reader->gram_scheme().has_value() && + current_gram_scheme != logical_reader->gram_scheme()) { + return Status::Error( + "gram index segment was cut with a different scheme than the current " + "analyzer; the predicate is evaluated without the index"); + } InvertedIndexQueryInfo execution_query_info = query_info; RETURN_IF_ERROR(_parse_query_terms(context, plain_analysis_str, query_type, analyzer_ctx, diff --git a/be/src/storage/index/snii/snii_index_reader.h b/be/src/storage/index/snii/snii_index_reader.h index 39ba9458a6284f..f187796f12358f 100644 --- a/be/src/storage/index/snii/snii_index_reader.h +++ b/be/src/storage/index/snii/snii_index_reader.h @@ -27,6 +27,10 @@ #include "storage/index/inverted/inverted_index_query_type.h" #include "storage/index/inverted/inverted_index_reader.h" +namespace doris::segment_v2::gram { +struct GramScheme; +} // namespace doris::segment_v2::gram + namespace doris::snii::reader { class LogicalIndexReader; } // namespace doris::snii::reader @@ -116,6 +120,8 @@ class SniiIndexReader final : public InvertedIndexReader { std::shared_ptr& bit_map, InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, const InvertedIndexAnalyzerCtx* analyzer_ctx); + Status _current_gram_scheme(const InvertedIndexAnalyzerCtx* analyzer_ctx, + std::optional* out) const; Status _parse_query_terms(const IndexQueryContextPtr& context, std::string search_str, InvertedIndexQueryType query_type, const InvertedIndexAnalyzerCtx* analyzer_ctx, diff --git a/be/test/exprs/function/like_gram_binding_test.cpp b/be/test/exprs/function/like_gram_binding_test.cpp index 9dc4ba4804b6e4..25eac0d927c80b 100644 --- a/be/test/exprs/function/like_gram_binding_test.cpp +++ b/be/test/exprs/function/like_gram_binding_test.cpp @@ -467,5 +467,51 @@ TEST_F(LikeGramBindingTest, GramPushDownDoesNoIndexIoWhenItsResultWouldBeDiscard } } +// An analyzed query (MATCH_*) is only exact on a gram segment when the current analyzer cuts +// it with the scheme that cut the segment. Here the physical index was written dense-3, while +// the index metadata now names a dense-4 analyzer (a recovered table whose policy was +// recreated; a calibrated density differs from the configured one the same way): the query +// must be skipped, not answered from grams that mean something else. Through the scheme that +// cut the segment the same query is answered by the index. +TEST_F(LikeGramBindingTest, MatchIsSkippedWhenTheCurrentSchemeDiffersFromTheSegments) { + TIndexPolicy tokenizer; + tokenizer.id = 6753832; + tokenizer.name = "like_binding_match_dense4_tokenizer"; + tokenizer.type = TIndexPolicyType::TOKENIZER; + tokenizer.properties = {{"type", "ngram"}, {"mode", "dense"}, {"min_gram", "4"}}; + TIndexPolicy analyzer; + analyzer.id = 6753833; + analyzer.name = "like_binding_match_dense4_analyzer"; + analyzer.type = TIndexPolicyType::ANALYZER; + analyzer.properties = {{"tokenizer", tokenizer.name}}; + _policy_mgr.apply_policy_changes({tokenizer, analyzer}, {}); + + TabletIndexPB index_pb; + _index_meta.to_schema_pb(&index_pb); + (*index_pb.mutable_properties())["analyzer"] = analyzer.name; + TabletIndex recovered; + recovered.init_from_pb(index_pb); + + const auto value = Field::create_field(std::string("abcdef")); + std::shared_ptr bitmap; + auto mismatched = segment_v2::SniiIndexReader::create_shared( + &recovered, _file_reader, segment_v2::InvertedIndexReaderType::FULLTEXT, _values.size(), + /*column_is_array=*/false); + for (const auto type : {segment_v2::InvertedIndexQueryType::MATCH_ANY_QUERY, + segment_v2::InvertedIndexQueryType::MATCH_ALL_QUERY}) { + Status status = mismatched->query(_query_context, "p", value, type, bitmap); + EXPECT_TRUE(status.is()) << status; + } + + auto same = segment_v2::SniiIndexReader::create_shared( + &_index_meta, _file_reader, segment_v2::InvertedIndexReaderType::FULLTEXT, + _values.size(), /*column_is_array=*/false); + Status status = same->query(_query_context, "p", value, + segment_v2::InvertedIndexQueryType::MATCH_ANY_QUERY, bitmap); + ASSERT_TRUE(status.ok()) << status; + ASSERT_NE(bitmap, nullptr); + EXPECT_FALSE(bitmap->isEmpty()) << "the segment's own scheme answers the query"; +} + } // namespace } // namespace doris diff --git a/regression-test/data/inverted_index_p0/gram/test_gram_policy_recovery.out b/regression-test/data/inverted_index_p0/gram/test_gram_policy_recovery.out index 387ef025ccd926..f8628e28138b27 100644 --- a/regression-test/data/inverted_index_p0/gram/test_gram_policy_recovery.out +++ b/regression-test/data/inverted_index_p0/gram/test_gram_policy_recovery.out @@ -13,6 +13,14 @@ 1 2 +-- !recovered_false_match_any -- +1 +2 + +-- !recovered_false_match_all -- +1 +2 + -- !recovered_true_like -- 1 2 @@ -21,6 +29,14 @@ 1 2 +-- !recovered_true_match_any -- +1 +2 + +-- !recovered_true_match_all -- +1 +2 + -- !mixed_false_like -- 1 2 @@ -43,6 +59,16 @@ 5 6 +-- !mixed_false_match_any -- +1 +2 +5 + +-- !mixed_false_match_all -- +1 +2 +5 + -- !mixed_true_like -- 1 2 @@ -65,3 +91,13 @@ 5 6 +-- !mixed_true_match_any -- +1 +2 +5 + +-- !mixed_true_match_all -- +1 +2 +5 + diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy index 04c5b106151913..18ba0bc50cc806 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy @@ -83,12 +83,19 @@ suite("test_gram_policy_recovery", "p0") { qt_dense4_tokens """SELECT TOKENIZE('abcdef', '"analyzer"="gram_recovery_analyzer"')""" sql "RECOVER TABLE test_gram_policy_recovery" + // MATCH terms come out of the current (dense4) analyzer while the recovered segment was + // cut dense3: the index must step aside for such queries and let the scalar predicate + // answer, so the two passes agree here as they do for LIKE / REGEXP. [false, true].each { useIndex -> sql "SET enable_inverted_index_query=${useIndex}" "order_qt_recovered_${useIndex}_like"("""SELECT id FROM test_gram_policy_recovery WHERE msg LIKE '%abcdef%'""") "order_qt_recovered_${useIndex}_regexp"("""SELECT id FROM test_gram_policy_recovery WHERE msg REGEXP 'abc.*def'""") + "order_qt_recovered_${useIndex}_match_any"("""SELECT id FROM test_gram_policy_recovery + WHERE msg MATCH_ANY 'abcdef'""") + "order_qt_recovered_${useIndex}_match_all"("""SELECT id FROM test_gram_policy_recovery + WHERE msg MATCH_ALL 'abcdef'""") } // This writer sees dense4, while the recovered rowset retains its dense3 dictionary. @@ -107,6 +114,12 @@ suite("test_gram_policy_recovery", "p0") { WHERE msg LIKE '%abc%'""") "order_qt_mixed_${useIndex}_short_regexp"("""SELECT id FROM test_gram_policy_recovery WHERE msg REGEXP 'abc'""") + // dense3 and dense4 segments side by side: the index answers MATCH on the segment + // whose scheme the current analyzer shares and steps aside on the other. + "order_qt_mixed_${useIndex}_match_any"("""SELECT id FROM test_gram_policy_recovery + WHERE msg MATCH_ANY 'abcdef'""") + "order_qt_mixed_${useIndex}_match_all"("""SELECT id FROM test_gram_policy_recovery + WHERE msg MATCH_ALL 'abcdef'""") } sql "SET enable_inverted_index_query=true" } From 356e0affade80f3ab3886c85600d846b819dd16a Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 11 Sep 2026 09:49:25 +0800 Subject: [PATCH 08/36] [fix](be) adaptive density: solve on the bytes the extractor cuts, for sparse schemes only, with defaults that can be met Seven corrections to the write-side density solve, each pinned by a unit test: - Sparse schemes only. Density is a sparse-scheme parameter; a dense writer used to hold back its rows, fill the histogram and replay the sample for a rate its extractor never reads. Dense writers cut rows as they arrive and charge no sample. - Defaults that can be met. GramScheme's default max_gram drops from 16 to 4 (the measured default: about 60% less dictionary than 16 with the posting lists unchanged), so a bare mode=sparse tokenizer can keep the 12-byte literal promise. A tokenizer whose max_gram exceeds the promise has the promise raised to max_gram, since a literal shorter than a gram cannot hold a whole one, instead of collecting no evidence. - One window per start position. The scan stopped at the run's last byte pair and counted max_gram - 2 windows extending past the run, biasing the quantile toward the tail; it now ends at pair run.size() - max_gram and counts exactly run.size() - min_literal + 1 windows. - NUL ends a run. The extractor emits no gram across a NUL byte, so a window across one is no evidence; the solver splits runs at NUL as it does at non-ASCII bytes. - The same bytes the extractor sees. A CHAR value is observed and retained up to its first NUL, as it is tokenized, and a lower_case scheme folds the evidence before hashing, as the extractor folds the row. - Honest accounting of the held-back sample. Each row is charged its payload plus the vector element and string header that hold it, so empty or tiny rows reach the cap by count; the solver's 256 KB histogram is charged to the memory reporter while it is resident. - A clamped solve is reported. When the required rate lies past the ceiling, solve_detailed says so together with the share the clamped rate really keeps, and the writer logs the unmet target instead of recording the ceiling as if it met the coverage. The three density configs are validated where they are set: gram_index_density_coverage_permille in (0, 1000], min_literal_bytes >= 1, sample_bytes >= 0. A negative permille used to reach the writer's checked cast and fail every gram-index load until the live config was repaired. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01METuP3aVRn8dfCU62PivnF --- be/src/common/config.cpp | 19 ++ .../index/inverted/gram/gram_density.cpp | 79 ++++-- .../index/inverted/gram/gram_density.h | 19 +- .../storage/index/inverted/gram/gram_scheme.h | 10 +- .../tokenizer/ngram/ngram_tokenizer_factory.h | 2 +- .../storage/index/snii/snii_index_writer.cpp | 56 ++++- be/src/storage/index/snii/snii_index_writer.h | 9 + .../index/inverted/gram/gram_density_test.cpp | 112 +++++++++ .../inverted/gram/gram_extractor_test.cpp | 3 +- .../index/inverted/gram/gram_scheme_test.cpp | 4 +- .../tokenizer/gram_tokenizer_test.cpp | 4 +- .../index/snii/format/core_metadata_test.cpp | 4 +- .../writer/gram_density_calibration_test.cpp | 225 +++++++++++++++++- be/test/storage/index/snii_writer_test.cpp | 2 +- 14 files changed, 499 insertions(+), 49 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 8f84bdc79cffa9..84cef1784fbbcc 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1356,6 +1356,20 @@ bool valid_gram_index_candidate_min_rows(int32_t value) { return value >= 0; } +bool valid_gram_index_min_literal_bytes(int32_t value) { + return value >= 1; +} + +// A share of windows: (0, 1000]. 0 would ask for nothing and a negative value used to reach +// the writer's checked cast and fail every gram-index load until the config was repaired. +bool valid_gram_index_density_coverage_permille(int32_t value) { + return value >= 1 && value <= 1000; +} + +bool valid_gram_index_density_sample_bytes(int64_t value) { + return value >= 0; +} + } // namespace // Whether LIKE/REGEXP tries to compile a constant pattern into a gram boolean query pushed down @@ -1374,10 +1388,15 @@ DEFINE_mInt32(gram_index_candidate_ratio_min_rows, "65536"); DEFINE_mBool(enable_gram_index_adaptive_density, "true"); // 12 bytes at 95% of windows. Both are the promise rather than a tuning pair -- what a user // may reasonably change is how short a literal they expect to find, not the boundary rate. +// A literal shorter than the scheme's max_gram cannot hold a whole gram, so the writer +// raises the promise to max_gram where a tokenizer sets a longer one. DEFINE_mInt32(gram_index_min_literal_bytes, "12"); DEFINE_mInt32(gram_index_density_coverage_permille, "950"); DEFINE_mInt64(gram_index_density_sample_bytes, "4194304"); DEFINE_Validator(gram_index_candidate_ratio_min_rows, valid_gram_index_candidate_min_rows); +DEFINE_Validator(gram_index_min_literal_bytes, valid_gram_index_min_literal_bytes); +DEFINE_Validator(gram_index_density_coverage_permille, valid_gram_index_density_coverage_permille); +DEFINE_Validator(gram_index_density_sample_bytes, valid_gram_index_density_sample_bytes); // condition cache limit DEFINE_Int16(condition_cache_limit, "512"); diff --git a/be/src/storage/index/inverted/gram/gram_density.cpp b/be/src/storage/index/inverted/gram/gram_density.cpp index 28627bdb46e938..f1d86c514b3ed6 100644 --- a/be/src/storage/index/inverted/gram/gram_density.cpp +++ b/be/src/storage/index/inverted/gram/gram_density.cpp @@ -23,11 +23,20 @@ namespace doris::segment_v2::gram { -DensitySolver::DensitySolver(size_t min_literal_len, size_t max_gram_len) +DensitySolver::DensitySolver(size_t min_literal_len, size_t max_gram_len, bool lower_case) : _min_literal_len(min_literal_len), _max_gram_len(max_gram_len), + _lower_case(lower_case), _histogram(kHashValues, 0) {} +namespace gram_density_detail { +// The bytes the extractor cuts grams from: ASCII, and never NUL -- a candidate gram that +// contains one is dropped, so a literal window that contains one can never be found. +inline bool indexable_byte(unsigned char c) { + return c < 0x80 && c != 0; +} +} // namespace gram_density_detail + void DensitySolver::observe(std::string_view value) { if (_min_literal_len < _max_gram_len || _max_gram_len < 2) { return; @@ -36,18 +45,27 @@ void DensitySolver::observe(std::string_view value) { // [k, k+max_gram_len) at worst, so k may not exceed len - max_gram_len. That leaves // `per_window` candidate positions in a window of the promised length. const size_t per_window = _min_literal_len - _max_gram_len + 1; + // The extractor folds ASCII case before it hashes when the scheme says so; the pair + // hashes that decide a boundary have to be computed over the same bytes here. + const auto byte_at = [&](std::string_view run, size_t k) -> uint8_t { + auto c = static_cast(run[k]); + if (_lower_case && c >= 'A' && c <= 'Z') { + c = static_cast(c - 'A' + 'a'); + } + return c; + }; size_t i = 0; const size_t n = value.size(); while (i < n) { - // Only ASCII runs: they are the only bytes the extractor indexes, so a window - // spanning anything else could never produce a gram. - if (static_cast(value[i]) >= 0x80) { + // Only runs of indexable bytes: a window spanning anything else could never produce + // a gram, so it carries no evidence either way. + if (!gram_density_detail::indexable_byte(static_cast(value[i]))) { ++i; continue; } size_t j = i; - while (j < n && static_cast(value[j]) < 0x80) { + while (j < n && gram_density_detail::indexable_byte(static_cast(value[j]))) { ++j; } const std::string_view run = value.substr(i, j - i); @@ -58,15 +76,19 @@ void DensitySolver::observe(std::string_view value) { // Sliding minimum over the run's pair hashes, one window per start position. The // monotonic queue keeps this linear in the run rather than quadratic in the window, // which is what makes it affordable on the write path. - const size_t pairs = run.size() - 1; + // + // A window starting at s spans [s, s + min_literal_len) and its candidate boundary + // positions are [s, s + per_window); the last real window therefore ends the scan at + // pair run.size() - max_gram_len, and exactly run.size() - min_literal_len + 1 windows + // are counted. Running the queue over the remaining pairs would count max_gram_len - 2 + // more windows that extend past the run, which biased the quantile toward the tail. + const size_t last_pair = run.size() - _max_gram_len; _mono.clear(); size_t head = 0; - for (size_t p = 0; p < pairs; ++p) { - const uint16_t h = - boundary_hash16(static_cast(run[p]), static_cast(run[p + 1])); - while (_mono.size() > head && - boundary_hash16(static_cast(run[_mono.back()]), - static_cast(run[_mono.back() + 1])) >= h) { + for (size_t p = 0; p <= last_pair; ++p) { + const uint16_t h = boundary_hash16(byte_at(run, p), byte_at(run, p + 1)); + while (_mono.size() > head && boundary_hash16(byte_at(run, _mono.back()), + byte_at(run, _mono.back() + 1)) >= h) { _mono.pop_back(); } _mono.push_back(static_cast(p)); @@ -77,19 +99,22 @@ void DensitySolver::observe(std::string_view value) { while (_mono[head] < window_start) { ++head; } - const uint16_t min_hash = boundary_hash16(static_cast(run[_mono[head]]), - static_cast(run[_mono[head] + 1])); + const uint16_t min_hash = + boundary_hash16(byte_at(run, _mono[head]), byte_at(run, _mono[head] + 1)); ++_histogram[min_hash]; ++_windows; } } } -uint16_t DensitySolver::solve(uint32_t coverage_permille) const { +DensitySolver::Solution DensitySolver::solve_detailed(uint32_t coverage_permille) const { + Solution out; if (_windows == 0 || coverage_permille == 0) { // No evidence, or nothing asked for. Stay where a configured default would have been // rather than inventing a rate from nothing. - return kMaxSolvedDensityPermille; + out.density_permille = kMaxSolvedDensityPermille; + out.required_permille = kMaxSolvedDensityPermille; + return out; } // A window is covered exactly when its minimum is below the threshold, so the prefix sum // of the histogram IS the coverage curve: walk it until the requested share is reached and @@ -107,10 +132,26 @@ uint16_t DensitySolver::solve(uint32_t coverage_permille) const { } // GramExtractor derives its threshold as density_permille * 65536 / 1000 with integer // division; round up or the resolved density can land a notch below what was solved for. - uint32_t permille = + out.required_permille = static_cast((static_cast(threshold) * 1000 + 65535) / 65536); - permille = std::clamp(permille, kMinSolvedDensityPermille, kMaxSolvedDensityPermille); - return static_cast(permille); + const uint32_t clamped = std::clamp(out.required_permille, kMinSolvedDensityPermille, + kMaxSolvedDensityPermille); + out.density_permille = static_cast(clamped); + out.clamped = clamped != out.required_permille; + // The share the clamped rate really keeps: the windows whose minimum sits below the + // threshold GramExtractor will derive from it. + const size_t clamped_threshold = std::min( + static_cast(static_cast(clamped) * 65536 / 1000), kHashValues); + uint64_t covered = 0; + for (size_t h = 0; h < clamped_threshold; ++h) { + covered += _histogram[h]; + } + out.achieved_coverage_permille = static_cast(covered * 1000 / _windows); + return out; +} + +uint16_t DensitySolver::solve(uint32_t coverage_permille) const { + return solve_detailed(coverage_permille).density_permille; } uint16_t solve_density_permille(const std::vector& sample_rows, size_t min_literal_len, diff --git a/be/src/storage/index/inverted/gram/gram_density.h b/be/src/storage/index/inverted/gram/gram_density.h index 64a0dd950c8814..fac59181b49a84 100644 --- a/be/src/storage/index/inverted/gram/gram_density.h +++ b/be/src/storage/index/inverted/gram/gram_density.h @@ -50,10 +50,13 @@ class DensitySolver { // fits after it, so the last max_gram_len-1 positions of a window cannot carry coverage. // Counting them made an earlier version optimistic -- it asked for 95% of 12-byte literals // on URL paths and delivered 86%. - DensitySolver(size_t min_literal_len, size_t max_gram_len); + // `lower_case` mirrors the scheme: the extractor folds ASCII case before it hashes byte + // pairs, so the evidence has to be folded the same way or it describes another corpus. + DensitySolver(size_t min_literal_len, size_t max_gram_len, bool lower_case = false); // Feeds one column value. Only ASCII runs contribute, since they are the only bytes the - // extractor indexes. + // extractor indexes, and a NUL byte ends a run the way a non-ASCII byte does: the + // extractor emits no gram across one, so no window across one can count as covered. void observe(std::string_view value); // Windows seen so far. Zero means nothing of the promised length was present and solve() @@ -64,6 +67,17 @@ class DensitySolver { // windows, clamped to the bounds above. With no evidence it returns the ceiling rather // than inventing a rate. uint16_t solve(uint32_t coverage_permille) const; + // What solve() clamps away. `required_permille` is the unclamped answer; when it lies + // outside the bounds, the clamped `density_permille` keeps only + // `achieved_coverage_permille` of the observed windows instead of the requested share, + // and the caller must say so rather than record the clamped rate as if it met the target. + struct Solution { + uint16_t density_permille = kMaxSolvedDensityPermille; + uint32_t required_permille = 0; + uint32_t achieved_coverage_permille = 0; + bool clamped = false; + }; + Solution solve_detailed(uint32_t coverage_permille) const; // Fixed regardless of how much has been observed. static constexpr size_t heap_bytes() { return kHashValues * sizeof(uint32_t); } @@ -73,6 +87,7 @@ class DensitySolver { size_t _min_literal_len; size_t _max_gram_len; + bool _lower_case; uint64_t _windows = 0; // Counts of per-window minimum hashes, indexed by the hash itself. std::vector _histogram; diff --git a/be/src/storage/index/inverted/gram/gram_scheme.h b/be/src/storage/index/inverted/gram/gram_scheme.h index 4797a2119decdd..a684ce7717ef06 100644 --- a/be/src/storage/index/inverted/gram/gram_scheme.h +++ b/be/src/storage/index/inverted/gram/gram_scheme.h @@ -37,8 +37,12 @@ enum class GramMode : uint8_t { DENSE = 1, SPARSE = 2 }; // cache key. struct GramScheme { GramMode mode = GramMode::SPARSE; - uint32_t min_len = 3; // n (bytes) - uint32_t max_len = 16; // L (bytes, SPARSE only) + uint32_t min_len = 3; // n (bytes) + // L (bytes, SPARSE only). 4 is the measured default: against 16 it cuts the dictionary + // by about 60% with the posting lists unchanged, and it keeps the adaptive density's + // 12-byte literal promise satisfiable -- a promise shorter than max_gram cannot be kept + // (see gram_density.h). + uint32_t max_len = 4; uint32_t density_permille = 250; // p x 1000 (SPARSE only) bool lower_case = false; uint32_t hash_version = 1; @@ -49,7 +53,7 @@ struct GramScheme { // Write back to a property table, for persisting the segment metadata and as the input to // the cache key computation. std::map to_properties() const; - // Cache key of the form "gram:v1:sparse:3:16:250:lc0", uniquely identifying one set of + // Cache key of the form "gram:v1:sparse:3:4:250:lc0", uniquely identifying one set of // scheme parameters. std::string cache_key() const; bool operator==(const GramScheme& o) const { diff --git a/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h b/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h index e74eac55356692..e325e9bc1197f5 100644 --- a/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h +++ b/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h @@ -81,7 +81,7 @@ class NGramTokenizerFactory : public TokenizerFactory { // parsing (R16, DRY). When "mode" is absent, *out is set to nullopt and OK is returned // (legacy ngram); on an illegal value (an unknown mode, an out-of-range min/max_gram, ...) // InvalidArgument is returned and *out stays nullopt. Every key that is absent falls back to - // GramScheme's own member initializer (min_gram=3, max_gram=16, ...). + // GramScheme's own member initializer (min_gram=3, max_gram=4, ...). static Status parse_gram_scheme(const Settings& settings, std::optional* out); private: diff --git a/be/src/storage/index/snii/snii_index_writer.cpp b/be/src/storage/index/snii/snii_index_writer.cpp index ce6943112f1e93..e44b79e986895a 100644 --- a/be/src/storage/index/snii/snii_index_writer.cpp +++ b/be/src/storage/index/snii/snii_index_writer.cpp @@ -82,11 +82,20 @@ void SniiIndexColumnWriter::_apply_gram_family_scheme( // what a segment gets when the feature is off, when the sample carries no window of the // promised length, or when nothing at all is written. void SniiIndexColumnWriter::_arm_density_calibration() { - if (!_gram_scheme.has_value() || !config::enable_gram_index_adaptive_density) { + // Density is a sparse-scheme parameter: a dense index cuts every position whatever the + // rate, so holding its rows back would settle nothing and cost every dense writer the + // sample, the histogram and a replay. + if (!_gram_scheme.has_value() || _gram_scheme->mode != gram::GramMode::SPARSE || + !config::enable_gram_index_adaptive_density) { return; } - const auto min_literal = static_cast(config::gram_index_min_literal_bytes); - _density_solver = std::make_unique(min_literal, _gram_scheme->max_len); + // A literal shorter than a gram can never hold a whole one, so the promise cannot be + // shorter than max_gram: a tokenizer with a longer max_gram solves for the shortest + // literal it can actually keep instead of collecting no evidence at all. + _density_promise_bytes = std::max( + static_cast(config::gram_index_min_literal_bytes), _gram_scheme->max_len); + _density_solver = std::make_unique( + _density_promise_bytes, _gram_scheme->max_len, _gram_scheme->lower_case); _density_calibrating = true; } @@ -284,11 +293,15 @@ Status SniiIndexColumnWriter::add_values(const std::string /*name*/, const void* const auto* v = reinterpret_cast(values); for (size_t i = 0; i < count; ++i) { if (_density_calibrating) { - // Held back, not dropped: this row is tokenized once the rate is known. Feeding - // the solver here costs one linear pass over the row and retains nothing of it. - _density_solver->observe(std::string_view(v->data, v->size)); - _density_sample.emplace_back(_rid, std::string(v->data, v->size)); - _density_sample_bytes += static_cast(v->size); + // Held back, not dropped: this row is tokenized once the rate is known. The + // solver sees exactly the bytes the extractor will see -- a CHAR value ends at + // its first NUL, as in _add_value_tokens -- and the sample is charged at what it + // really retains: the payload plus the vector element and string header that + // hold it, so a run of empty or tiny rows still reaches the cap. + const size_t logical_size = _is_char ? strnlen(v->data, v->size) : v->size; + _density_solver->observe(std::string_view(v->data, logical_size)); + _density_sample.emplace_back(_rid, std::string(v->data, logical_size)); + _density_sample_bytes += static_cast(logical_size) + kDensitySampleRowOverhead; _report_density_sample_capacity(); ++v; ++_rid; @@ -451,11 +464,22 @@ Status SniiIndexColumnWriter::_finish_density_calibration() { return Status::OK(); } _density_calibrating = false; - const uint16_t solved = _density_solver->solve( - cast_set(config::gram_index_density_coverage_permille)); + const auto coverage = cast_set(config::gram_index_density_coverage_permille); + const gram::DensitySolver::Solution solution = _density_solver->solve_detailed(coverage); if (_density_solver->observed_windows() > 0) { - _gram_scheme->density_permille = solved; - _pending_density_permille = solved; + _gram_scheme->density_permille = solution.density_permille; + _pending_density_permille = solution.density_permille; + if (solution.clamped && solution.achieved_coverage_permille < coverage) { + // The sample asked for a rate outside the solver's bounds, so the recorded rate + // does not keep the promise. Say so: a segment that silently misses its target + // reads as an index that does not work on literals it was supposed to find. + LOG(WARNING) << "gram index " << _index_meta->index_id() + << ": adaptive density clamped to " << solution.density_permille + << " permille; " << solution.required_permille + << " permille would be needed to cover " << coverage << " permille of " + << _density_promise_bytes << "-byte literals, the clamped rate covers " + << solution.achieved_coverage_permille << " permille"; + } } // Free the histogram before replaying: it is 256 KB that nothing needs again. _density_solver.reset(); @@ -482,7 +506,13 @@ void SniiIndexColumnWriter::_report_density_sample_capacity(bool release_all) { if (_memory_reporter == nullptr) { return; } - const int64_t now = release_all ? 0 : _density_sample_bytes; + // The solver's histogram is resident for exactly as long as the sample is. + const int64_t now = + release_all ? 0 + : _density_sample_bytes + + (_density_solver != nullptr + ? static_cast(gram::DensitySolver::heap_bytes()) + : 0); if (now != _density_sample_charged_bytes) { _memory_reporter->report(now - _density_sample_charged_bytes); _density_sample_charged_bytes = now; diff --git a/be/src/storage/index/snii/snii_index_writer.h b/be/src/storage/index/snii/snii_index_writer.h index e895d57f4d79b6..17b0274f6700eb 100644 --- a/be/src/storage/index/snii/snii_index_writer.h +++ b/be/src/storage/index/snii/snii_index_writer.h @@ -83,6 +83,9 @@ class SniiIndexColumnWriter final : public IndexColumnWriter { ::doris::snii::format::IndexConfig config_for_test() const { return _config; } bool writes_norms_for_test() const { return _writes_norms; } const std::optional& gram_scheme_for_test() const { return _gram_scheme; } + bool density_calibrating_for_test() const { return _density_calibrating; } + int64_t density_sample_bytes_for_test() const { return _density_sample_bytes; } + size_t density_promise_bytes_for_test() const { return _density_promise_bytes; } void set_analysis_for_test(inverted_index::ReaderPtr reader, std::shared_ptr analyzer) { _should_analyzer = true; @@ -162,7 +165,13 @@ class SniiIndexColumnWriter final : public IndexColumnWriter { // solved, applied, and the held-back rows are tokenized with it. std::unique_ptr _density_solver; std::vector> _density_sample; + // Per held-back row: the vector element and the string header it carries, on top of the + // payload. Charged so that the sample cap is reached by row count as well as by bytes. + static constexpr int64_t kDensitySampleRowOverhead = + static_cast(sizeof(std::pair)); int64_t _density_sample_bytes = 0; + // The literal length the solve promises, after the max_gram floor was applied. + size_t _density_promise_bytes = 0; int64_t _density_sample_charged_bytes = 0; bool _density_calibrating = false; // Set when the rate has been solved but the tokenizer has not been obtained yet; applied diff --git a/be/test/storage/index/inverted/gram/gram_density_test.cpp b/be/test/storage/index/inverted/gram/gram_density_test.cpp index a5109b6a1aa900..507940dcdbdbd9 100644 --- a/be/test/storage/index/inverted/gram/gram_density_test.cpp +++ b/be/test/storage/index/inverted/gram/gram_density_test.cpp @@ -165,4 +165,116 @@ TEST(GramDensityTest, DegenerateInputFallsBackWithinBounds) { EXPECT_LE(solve_density_permille(log_like_rows(), 12, 4, 1000), kMaxSolvedDensityPermille); } +// The byte whose self-pair hashes highest among printable ASCII: rows made of it alone put +// every window's minimum at that hash, which is far above the ceiling's threshold. +namespace { +unsigned char highest_self_pair_byte() { + unsigned char best = '!'; + for (int c = '!'; c <= '~'; ++c) { + const auto b = static_cast(c); + if (boundary_hash16(b, b) > boundary_hash16(best, best)) { + best = b; + } + } + return best; +} +} // namespace + +// Exactly one window per start position: a run of n bytes holds n - L + 1 windows of length L +// whatever max_gram is. The scan used to run through every byte pair and count max_gram - 2 +// windows that extend past the run's end. +TEST(GramDensityTest, CountsExactlyOneWindowPerStartPosition) { + for (const size_t max_gram : std::vector {2, 3, 4, 8}) { + for (const size_t len : std::vector {24, 25, 40}) { + DensitySolver solver(12, max_gram); + std::string row(len, 'a'); + for (size_t i = 0; i < len; ++i) { + row[i] = static_cast('a' + (i * 7) % 26); + } + solver.observe(row); + EXPECT_EQ(solver.observed_windows(), len - 12 + 1) + << "len " << len << " max_gram " << max_gram; + } + } + DensitySolver one(12, 4); + one.observe("abcdefghijkl"); + EXPECT_EQ(one.observed_windows(), 1U) << "a run as long as the promise is one window"; + DensitySolver none(12, 4); + none.observe("abcdefghijk"); + EXPECT_EQ(none.observed_windows(), 0U) << "a shorter run is no window at all"; +} + +// NUL ends a run the way a non-ASCII byte does: the extractor never emits a gram across one, +// so a window across one is not evidence of anything findable. +TEST(GramDensityTest, NulEndsARunLikeNonAscii) { + DensitySolver with_nul(3, 2); + with_nul.observe(std::string("A\0B", 3)); + EXPECT_EQ(with_nul.observed_windows(), 0U); + + DensitySolver split(12, 4); + std::string row = "abcdefghijklmn"; // 14 bytes: 3 windows + row.push_back('\0'); + row += "opqrstuvwxyzabc"; // 15 bytes: 4 windows + split.observe(row); + EXPECT_EQ(split.observed_windows(), 3U + 4U) << "not the 19 of an unbroken 30-byte run"; +} + +// With lower_case the evidence is folded exactly as the extractor folds the row before it +// hashes, so upper-case rows through a folding solver solve to what their lower-case twins +// solve to through a plain one. +TEST(GramDensityTest, FoldsCaseWhenTheSchemeDoes) { + std::vector lower = log_like_rows(); + for (std::string& row : lower) { + for (char& ch : row) { + if (ch >= 'A' && ch <= 'Z') { + ch = static_cast(ch - 'A' + 'a'); + } + } + } + std::vector upper = lower; + for (std::string& row : upper) { + for (char& ch : row) { + if (ch >= 'a' && ch <= 'z') { + ch = static_cast(ch - 'a' + 'A'); + } + } + } + DensitySolver folded(12, 4, /*lower_case=*/true); + DensitySolver plain(12, 4, /*lower_case=*/false); + for (size_t i = 0; i < lower.size(); ++i) { + folded.observe(upper[i]); + plain.observe(lower[i]); + } + EXPECT_EQ(folded.observed_windows(), plain.observed_windows()); + EXPECT_EQ(folded.solve(950), plain.solve(950)); +} + +// A corpus whose quantile lies past the ceiling is clamped, and the solution says so together +// with the share the clamped rate really keeps -- here none, since the only pair hash in these +// rows is above the ceiling's threshold. An ordinary corpus is neither clamped nor short. +TEST(GramDensityTest, ReportsAClampedSolveAndItsRealCoverage) { + const unsigned char b = highest_self_pair_byte(); + ASSERT_GE(boundary_hash16(b, b), 32768U) + << "the test needs a self-pair above the ceiling's threshold"; + DensitySolver flat(12, 4); + for (int i = 0; i < 50; ++i) { + flat.observe(std::string(40, static_cast(b))); + } + const DensitySolver::Solution s = flat.solve_detailed(950); + EXPECT_TRUE(s.clamped); + EXPECT_EQ(s.density_permille, kMaxSolvedDensityPermille); + EXPECT_GT(s.required_permille, kMaxSolvedDensityPermille); + EXPECT_EQ(s.achieved_coverage_permille, 0U); + EXPECT_EQ(flat.solve(950), kMaxSolvedDensityPermille); + + DensitySolver ordinary(12, 4); + for (const std::string& row : log_like_rows()) { + ordinary.observe(row); + } + const DensitySolver::Solution t = ordinary.solve_detailed(950); + EXPECT_FALSE(t.clamped); + EXPECT_GE(t.achieved_coverage_permille, 950U); + EXPECT_EQ(t.density_permille, t.required_permille); +} + } // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/gram_extractor_test.cpp b/be/test/storage/index/inverted/gram/gram_extractor_test.cpp index 65363ac6890362..9ef1521d1c3837 100644 --- a/be/test/storage/index/inverted/gram/gram_extractor_test.cpp +++ b/be/test/storage/index/inverted/gram/gram_extractor_test.cpp @@ -101,6 +101,7 @@ TEST(GramExtractorTest, SparseGoldenFromPrototype) { // The golden was generated by // tools/regex-ngram-model/ngram_model.cpp --cdc --p 0.25 --maxlen 16 --n 3 GramScheme s; + s.max_len = 16; // the golden predates the default of 4 EXPECT_EQ(run(s, "rpc error: code = Unavailable"), (std::vector {"or: co", "cod", "ode = U", " Unavai", "ailable"})); } @@ -249,7 +250,7 @@ TEST(GramExtractorPerformanceTest, ShortRowsHaveBoundedConstructionOverhead) { for (GramMode mode : {GramMode::DENSE, GramMode::SPARSE}) { GramScheme scheme; scheme.mode = mode; - const size_t expected_grams = mode == GramMode::DENSE ? 120 : 27; + const size_t expected_grams = mode == GramMode::DENSE ? 120 : 28; std::array ratios {}; for (double& ratio : ratios) { size_t fresh_count = 0; diff --git a/be/test/storage/index/inverted/gram/gram_scheme_test.cpp b/be/test/storage/index/inverted/gram/gram_scheme_test.cpp index 75d732310990bd..bef19b4f346a53 100644 --- a/be/test/storage/index/inverted/gram/gram_scheme_test.cpp +++ b/be/test/storage/index/inverted/gram/gram_scheme_test.cpp @@ -29,12 +29,12 @@ TEST(GramSchemeTest, DefaultsAndRoundTrip) { GramScheme s; EXPECT_EQ(s.mode, GramMode::SPARSE); EXPECT_EQ(s.min_len, 3U); - EXPECT_EQ(s.max_len, 16U); + EXPECT_EQ(s.max_len, 4U); EXPECT_EQ(s.density_permille, 250U); GramScheme back; ASSERT_TRUE(GramScheme::from_properties(s.to_properties(), &back).ok()); EXPECT_TRUE(s == back); - EXPECT_EQ(s.cache_key(), "gram:v1:sparse:3:16:250:lc0"); + EXPECT_EQ(s.cache_key(), "gram:v1:sparse:3:4:250:lc0"); } TEST(GramSchemeTest, ParsesTokenizerProperties) { diff --git a/be/test/storage/index/inverted/tokenizer/gram_tokenizer_test.cpp b/be/test/storage/index/inverted/tokenizer/gram_tokenizer_test.cpp index 32e6b0853b85cb..2e9526627059c2 100644 --- a/be/test/storage/index/inverted/tokenizer/gram_tokenizer_test.cpp +++ b/be/test/storage/index/inverted/tokenizer/gram_tokenizer_test.cpp @@ -98,7 +98,7 @@ TEST(GramTokenizerTest, DenseModeAndAutoAlias) { EXPECT_EQ(f2.gram_scheme()->mode, gram::GramMode::SPARSE); // P0: auto = sparse } -// When min/max_gram are absent the defaults come from GramScheme's member initializers (3/16); +// When min/max_gram are absent the defaults come from GramScheme's member initializers (3/4); // the factory no longer injects a copy of its own. TEST(GramTokenizerTest, GramDefaultsComeFromGramScheme) { NGramTokenizerFactory factory; @@ -106,7 +106,7 @@ TEST(GramTokenizerTest, GramDefaultsComeFromGramScheme) { factory.initialize(Settings(args)); ASSERT_TRUE(factory.gram_scheme().has_value()); EXPECT_EQ(factory.gram_scheme()->min_len, 3U); - EXPECT_EQ(factory.gram_scheme()->max_len, 16U); + EXPECT_EQ(factory.gram_scheme()->max_len, 4U); } // With lower_case=true the folding happens before the split (the boundary hash is computed over diff --git a/be/test/storage/index/snii/format/core_metadata_test.cpp b/be/test/storage/index/snii/format/core_metadata_test.cpp index c0476e5aeee508..0d4f6338acb104 100644 --- a/be/test/storage/index/snii/format/core_metadata_test.cpp +++ b/be/test/storage/index/snii/format/core_metadata_test.cpp @@ -336,14 +336,14 @@ TEST(SniiCoreMetadata, GramSchemeRoundTrip) { TEST(SniiCoreMetadata, GramSchemeSparseRoundTrip) { auto expected = sample_core(); - expected.gram_scheme = GramScheme {}; // member defaults: SPARSE / 3 / 16 / 250 / lc0 / v1 + expected.gram_scheme = GramScheme {}; // member defaults: SPARSE / 3 / 4 / 250 / lc0 / v1 CoreMetadata actual; ASSERT_TRUE(decode_core_metadata(Slice(encode(expected)), &actual).ok()); ASSERT_TRUE(actual.gram_scheme.has_value()); EXPECT_EQ(actual.gram_scheme->mode, GramMode::SPARSE); EXPECT_EQ(actual.gram_scheme->min_len, 3U); - EXPECT_EQ(actual.gram_scheme->max_len, 16U); + EXPECT_EQ(actual.gram_scheme->max_len, 4U); EXPECT_EQ(actual.gram_scheme->density_permille, 250U); expect_core_eq(expected, actual); } diff --git a/be/test/storage/index/snii/writer/gram_density_calibration_test.cpp b/be/test/storage/index/snii/writer/gram_density_calibration_test.cpp index 6e2050c3a082b7..19ae40f60a55bc 100644 --- a/be/test/storage/index/snii/writer/gram_density_calibration_test.cpp +++ b/be/test/storage/index/snii/writer/gram_density_calibration_test.cpp @@ -26,7 +26,9 @@ #include #include +#include #include +#include #include #include "common/config.h" @@ -34,6 +36,7 @@ #include "runtime/exec_env.h" #include "runtime/index_policy/index_policy_mgr.h" #include "storage/index/inverted/gram/gram_density.h" +#include "storage/index/inverted/gram/gram_extractor.h" #include "storage/index/snii/snii_index_writer.h" #include "storage/tablet/tablet_schema.h" #include "util/defer_op.h" @@ -66,7 +69,40 @@ class ScopedGramPolicies { analyzer.name = "dens_cal_analyzer"; analyzer.type = TIndexPolicyType::ANALYZER; analyzer.properties["tokenizer"] = "dens_cal_tokenizer"; - _manager.apply_policy_changes({tokenizer, analyzer}, {}); + + // Variants for the tests below: a dense scheme, a bare sparse one (every optional + // property left to its default), a sparse one whose max_gram exceeds the promised + // literal length, and a folding one that is otherwise the base scheme. + std::vector policies {tokenizer, analyzer}; + const auto add = [&](int64_t id, const std::string& name, + std::map props) { + TIndexPolicy t; + t.id = id; + t.name = name + "_tokenizer"; + t.type = TIndexPolicyType::TOKENIZER; + t.properties["type"] = "ngram"; + for (auto& [k, v] : props) { + t.properties[k] = v; + } + TIndexPolicy a; + a.id = id + 1; + a.name = name + "_analyzer"; + a.type = TIndexPolicyType::ANALYZER; + a.properties["tokenizer"] = t.name; + policies.push_back(t); + policies.push_back(a); + }; + add(9403, "dens_cal_dense", {{"mode", "dense"}, {"min_gram", "3"}}); + add(9405, "dens_cal_bare", {{"mode", "sparse"}}); + add(9407, "dens_cal_long", + {{"mode", "sparse"}, {"min_gram", "3"}, {"max_gram", "16"}, {"density", "0.500"}}); + add(9409, "dens_cal_lc", + {{"mode", "sparse"}, + {"min_gram", "3"}, + {"max_gram", "4"}, + {"density", "0.500"}, + {"lower_case", "true"}}); + _manager.apply_policy_changes(policies, {}); } ~ScopedGramPolicies() { ExecEnv::GetInstance()->_index_policy_mgr = _previous; } @@ -75,13 +111,13 @@ class ScopedGramPolicies { IndexPolicyMgr* _previous = nullptr; }; -TabletIndex make_index_meta() { +TabletIndex make_index_meta(const std::string& analyzer = "dens_cal_analyzer") { TabletIndex meta; TabletIndexPB pb; pb.set_index_id(4401); pb.set_index_name("idx_dens"); pb.set_index_type(IndexType::INVERTED); - (*pb.mutable_properties())["analyzer"] = "dens_cal_analyzer"; + (*pb.mutable_properties())["analyzer"] = analyzer; (*pb.mutable_properties())["support_phrase"] = "false"; meta.init_from_pb(pb); return meta; @@ -201,4 +237,187 @@ TEST(GramDensityCalibrationTest, TheConfiguredRateStandsWhenTheSolveIsOff) { EXPECT_EQ(writer.gram_scheme_for_test()->density_permille, 500); } +// A dense scheme cuts every position whatever the rate, so nothing is held back for it: its +// rows are cut as they arrive and no sample is charged. +TEST(GramDensityCalibrationTest, DenseSchemesAreNotCalibrated) { + ScopedGramPolicies policies; + const bool saved = config::enable_gram_index_adaptive_density; + config::enable_gram_index_adaptive_density = true; + Defer restore([&] { config::enable_gram_index_adaptive_density = saved; }); + + TabletIndex meta = make_index_meta("dens_cal_dense_analyzer"); + SniiIndexColumnWriter writer(nullptr, &meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_TRUE(writer.init().ok()); + ASSERT_TRUE(writer.gram_scheme_for_test().has_value()); + EXPECT_FALSE(writer.density_calibrating_for_test()); + const std::vector data = rows(4); + const std::vector s = slices_of(data); + ASSERT_TRUE(writer.add_values("c", s.data(), s.size()).ok()); + EXPECT_GT(term_count(&writer), 0U) << "dense rows are cut as they arrive"; + EXPECT_EQ(writer.density_sample_bytes_for_test(), 0); +} + +// The shipped defaults collect evidence: a tokenizer with nothing but mode=sparse gets +// max_gram 4, the promise stays at the configured 12 bytes and a solve really happens. A +// tokenizer whose max_gram exceeds the promise has the promise raised to it, since a literal +// shorter than a gram can never hold a whole one. +TEST(GramDensityCalibrationTest, DefaultsCollectEvidenceAndALongMaxGramRaisesThePromise) { + ScopedGramPolicies policies; + const bool saved = config::enable_gram_index_adaptive_density; + const int64_t saved_budget = config::gram_index_density_sample_bytes; + config::enable_gram_index_adaptive_density = true; + config::gram_index_density_sample_bytes = 4096; + Defer restore([&] { + config::enable_gram_index_adaptive_density = saved; + config::gram_index_density_sample_bytes = saved_budget; + }); + { + TabletIndex meta = make_index_meta("dens_cal_bare_analyzer"); + SniiIndexColumnWriter writer(nullptr, &meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_TRUE(writer.init().ok()); + ASSERT_TRUE(writer.gram_scheme_for_test().has_value()); + EXPECT_EQ(writer.gram_scheme_for_test()->max_len, 4U); + EXPECT_EQ(writer.gram_scheme_for_test()->density_permille, 250U); + EXPECT_TRUE(writer.density_calibrating_for_test()); + EXPECT_EQ(writer.density_promise_bytes_for_test(), 12U); + // Rows of one repeated byte carry a single pair hash; picking the printable byte whose + // self-pair hashes highest forces the solve to the ceiling, which the configured + // default is not, so a scheme still at the default would mean no evidence reached it. + unsigned char b = '!'; + for (int c = '!'; c <= '~'; ++c) { + const auto cand = static_cast(c); + if (gram::boundary_hash16(cand, cand) > gram::boundary_hash16(b, b)) { + b = cand; + } + } + ASSERT_GE(gram::boundary_hash16(b, b), 32768U); + const std::vector flat(200, std::string(40, static_cast(b))); + const std::vector s = slices_of(flat); + ASSERT_TRUE(writer.add_values("c", s.data(), s.size()).ok()); + EXPECT_FALSE(writer.density_calibrating_for_test()); + EXPECT_EQ(writer.gram_scheme_for_test()->density_permille, gram::kMaxSolvedDensityPermille) + << "the solve ran on the defaults and reached the ceiling this corpus demands"; + } + { + TabletIndex meta = make_index_meta("dens_cal_long_analyzer"); + SniiIndexColumnWriter writer(nullptr, &meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_TRUE(writer.init().ok()); + EXPECT_TRUE(writer.density_calibrating_for_test()); + EXPECT_EQ(writer.density_promise_bytes_for_test(), 16U) + << "max_gram 16 cannot keep a 12-byte promise; the promise is raised to it"; + } +} + +// Empty rows retain a vector element and a string header each; charged for that, a run of +// them reaches the sample cap instead of growing the sample without bound. +TEST(GramDensityCalibrationTest, EmptyRowsStillReachTheSampleCap) { + ScopedGramPolicies policies; + const bool saved = config::enable_gram_index_adaptive_density; + const int64_t saved_budget = config::gram_index_density_sample_bytes; + config::enable_gram_index_adaptive_density = true; + config::gram_index_density_sample_bytes = 4096; + Defer restore([&] { + config::enable_gram_index_adaptive_density = saved; + config::gram_index_density_sample_bytes = saved_budget; + }); + + TabletIndex meta = make_index_meta(); + SniiIndexColumnWriter writer(nullptr, &meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_TRUE(writer.init().ok()); + ASSERT_TRUE(writer.density_calibrating_for_test()); + const std::vector empty(400, ""); + const std::vector s = slices_of(empty); + ASSERT_TRUE(writer.add_values("c", s.data(), s.size()).ok()); + EXPECT_FALSE(writer.density_calibrating_for_test()) + << "400 empty rows retain more than 4096 bytes of vector state"; + EXPECT_EQ(writer.gram_scheme_for_test()->density_permille, 500) + << "no window of the promised length: the configured rate stands unchanged"; +} + +// The solver and the sample see the bytes the extractor will see: a CHAR value ends at its +// first NUL, and its padding is neither evidence nor retained. +TEST(GramDensityCalibrationTest, CharPaddingIsNeitherEvidenceNorRetained) { + ScopedGramPolicies policies; + const bool saved = config::enable_gram_index_adaptive_density; + config::enable_gram_index_adaptive_density = true; + Defer restore([&] { config::enable_gram_index_adaptive_density = saved; }); + + TabletIndex meta = make_index_meta(); + SniiIndexColumnWriter writer(nullptr, &meta, FieldType::OLAP_FIELD_TYPE_CHAR); + ASSERT_TRUE(writer.init().ok()); + std::string padded = "rpc error: code = Unavailable"; + const auto logical = static_cast(padded.size()); + padded.resize(64, '\0'); + const doris::Slice slice(padded.data(), padded.size()); + ASSERT_TRUE(writer.add_values("c", &slice, 1).ok()); + EXPECT_EQ(writer.density_sample_bytes_for_test(), + logical + static_cast(sizeof(std::pair))); +} + +// A folding scheme folds its evidence: upper-case rows through it solve to what their +// lower-case twins solve to through the otherwise identical plain scheme. +TEST(GramDensityCalibrationTest, FoldingSchemesFoldTheEvidence) { + ScopedGramPolicies policies; + const bool saved = config::enable_gram_index_adaptive_density; + const int64_t saved_budget = config::gram_index_density_sample_bytes; + config::enable_gram_index_adaptive_density = true; + config::gram_index_density_sample_bytes = 4096; + Defer restore([&] { + config::enable_gram_index_adaptive_density = saved; + config::gram_index_density_sample_bytes = saved_budget; + }); + + std::vector lower = rows(200); + for (std::string& row : lower) { + for (char& ch : row) { + if (ch >= 'A' && ch <= 'Z') { + ch = static_cast(ch - 'A' + 'a'); + } + } + } + std::vector upper = lower; + for (std::string& row : upper) { + for (char& ch : row) { + if (ch >= 'a' && ch <= 'z') { + ch = static_cast(ch - 'a' + 'A'); + } + } + } + TabletIndex folding_meta = make_index_meta("dens_cal_lc_analyzer"); + SniiIndexColumnWriter folding(nullptr, &folding_meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_TRUE(folding.init().ok()); + const std::vector u = slices_of(upper); + ASSERT_TRUE(folding.add_values("c", u.data(), u.size()).ok()); + + TabletIndex plain_meta = make_index_meta(); + SniiIndexColumnWriter plain(nullptr, &plain_meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_TRUE(plain.init().ok()); + const std::vector l = slices_of(lower); + ASSERT_TRUE(plain.add_values("c", l.data(), l.size()).ok()); + + ASSERT_FALSE(folding.density_calibrating_for_test()); + ASSERT_FALSE(plain.density_calibrating_for_test()); + EXPECT_EQ(folding.gram_scheme_for_test()->density_permille, + plain.gram_scheme_for_test()->density_permille); +} + +// The coverage share and its neighbours are validated where they are set: a negative or +// impossible value is refused instead of reaching the writer's checked cast on every load. +TEST(GramDensityCalibrationTest, ConfigValidatorsRejectImpossibleValues) { + const auto coverage = std::to_string(config::gram_index_density_coverage_permille); + const auto literal = std::to_string(config::gram_index_min_literal_bytes); + const auto sample = std::to_string(config::gram_index_density_sample_bytes); + EXPECT_FALSE(config::set_config("gram_index_density_coverage_permille", "-1").ok()); + EXPECT_FALSE(config::set_config("gram_index_density_coverage_permille", "0").ok()); + EXPECT_FALSE(config::set_config("gram_index_density_coverage_permille", "1001").ok()); + EXPECT_TRUE(config::set_config("gram_index_density_coverage_permille", "950").ok()); + EXPECT_FALSE(config::set_config("gram_index_min_literal_bytes", "0").ok()); + EXPECT_TRUE(config::set_config("gram_index_min_literal_bytes", "12").ok()); + EXPECT_FALSE(config::set_config("gram_index_density_sample_bytes", "-1").ok()); + EXPECT_TRUE(config::set_config("gram_index_density_sample_bytes", "4194304").ok()); + EXPECT_TRUE(config::set_config("gram_index_density_coverage_permille", coverage).ok()); + EXPECT_TRUE(config::set_config("gram_index_min_literal_bytes", literal).ok()); + EXPECT_TRUE(config::set_config("gram_index_density_sample_bytes", sample).ok()); +} + } // namespace doris::segment_v2 diff --git a/be/test/storage/index/snii_writer_test.cpp b/be/test/storage/index/snii_writer_test.cpp index 1d3dc507c6dc13..f7f1c9136fadb0 100644 --- a/be/test/storage/index/snii_writer_test.cpp +++ b/be/test/storage/index/snii_writer_test.cpp @@ -583,7 +583,7 @@ TEST(SniiWriterTest, GramTokenizerForcesDocsOnlyAndIsRecognised) { } std::ranges::sort(terms); // Non-ASCII text contributes no terms: only the ASCII segment of the row is indexed. - std::vector expected {" Unavai", "ailable", "cod", "ode = U", "or: co"}; + std::vector expected {" Una", "aila", "cod", "ode ", "or: "}; std::ranges::sort(expected); EXPECT_EQ(terms, expected); } From 91b7021d9af82dfec84818fd8df5b81b35e46f14 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 11 Sep 2026 09:49:25 +0800 Subject: [PATCH 09/36] [fix](be) fence dropped-posting dictionaries behind a required SNII feature A dictionary entry whose posting list was dropped is flags plus df, with no locator. A reader that predates the flag ignores bit 5, reads the cleared low bits as a slim reference and then reads locator fields that are not there: it accepts the container and reports it as corrupt, and whether the query falls back to a scan depends on that incidental error path. The metadata directory now raises kFeatureDroppedPostings (2) in required_features whenever an inverted entry dropped at least one posting list. A reader without the feature refuses the container as unsupported up front, exactly as it does for the blob feature, and the query is evaluated without the index. Directories without such an entry keep their pre-feature bytes. The logical index writer counts its dropped entries and the compound writer carries the count into the directory entry. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01METuP3aVRn8dfCU62PivnF --- .../index/snii/format/format_constants.h | 5 +++++ .../index/snii/format/metadata_directory.cpp | 9 ++++++++ .../index/snii/format/metadata_directory.h | 3 +++ .../snii/writer/logical_index_writer.cpp | 1 + .../index/snii/writer/logical_index_writer.h | 4 ++++ .../snii/writer/snii_compound_writer.cpp | 3 +++ .../snii/format/metadata_directory_test.cpp | 21 +++++++++++++++++++ 7 files changed, 46 insertions(+) diff --git a/be/src/storage/index/snii/format/format_constants.h b/be/src/storage/index/snii/format/format_constants.h index 327ab990ecff9e..26079557c1a04f 100644 --- a/be/src/storage/index/snii/format/format_constants.h +++ b/be/src/storage/index/snii/format/format_constants.h @@ -39,6 +39,11 @@ inline constexpr uint16_t kMinReaderVersion = 1; // reject unknown ones as Unsupported, so a pure-text directory stays // byte-identical to the pre-blob format. inline constexpr uint32_t kFeatureBlobLogicalIndex = 1; +// At least one inverted entry holds dictionary entries whose posting list was dropped +// (dict_flags::kPostingDropped, stop-gram). Such an entry ends right after its df, which a +// reader that predates the flag would parse as a truncated locator; declaring the feature +// makes that reader refuse the container as unsupported instead of reporting corruption. +inline constexpr uint32_t kFeatureDroppedPostings = 2; // ---- SectionFramer type ids for standalone metadata blobs ---- enum class SectionType : uint8_t { diff --git a/be/src/storage/index/snii/format/metadata_directory.cpp b/be/src/storage/index/snii/format/metadata_directory.cpp index 01fe5dd6221cd3..75d061dea9615c 100644 --- a/be/src/storage/index/snii/format/metadata_directory.cpp +++ b/be/src/storage/index/snii/format/metadata_directory.cpp @@ -124,6 +124,10 @@ Status decode_directory_pb(const doris::snii::SniiMetadataDirectoryPB& input, feature_blob = true; continue; } + if (feature == kFeatureDroppedPostings) { + // Understood by this reader's dictionary decoder; nothing to validate here. + continue; + } return metadata_directory_unsupported( "metadata directory: required feature is not supported"); } @@ -226,6 +230,7 @@ Status encode_metadata_directory(const std::vector& ent doris::snii::SniiMetadataDirectoryPB directory; bool any_blob = false; + bool any_dropped = false; for (const auto& entry : entries) { auto* index = directory.add_indexes(); index->set_index_id(entry.index_id); @@ -241,6 +246,7 @@ Status encode_metadata_directory(const std::vector& ent encode_blob_ref(entry.core_metadata, index->mutable_core_metadata()); encode_blob_ref(entry.sampled_term_index, index->mutable_sampled_term_index()); encode_blob_ref(entry.dict_block_directory, index->mutable_dict_block_directory()); + any_dropped = any_dropped || entry.dropped_postings; } else { // The three inverted refs do not serialize for blob entries, so // the shared decode self-check below could not catch a caller that @@ -265,6 +271,9 @@ Status encode_metadata_directory(const std::vector& ent if (any_blob) { directory.add_required_features(kFeatureBlobLogicalIndex); } + if (any_dropped) { + directory.add_required_features(kFeatureDroppedPostings); + } std::vector validated; RETURN_IF_ERROR(decode_directory_pb(directory, &validated)); diff --git a/be/src/storage/index/snii/format/metadata_directory.h b/be/src/storage/index/snii/format/metadata_directory.h index 6bbd4d7d57d91b..c012e887091dee 100644 --- a/be/src/storage/index/snii/format/metadata_directory.h +++ b/be/src/storage/index/snii/format/metadata_directory.h @@ -65,6 +65,9 @@ struct LogicalIndexMetadataRef { // original members so existing designated-initializer sites stay valid. LogicalIndexKind kind = LogicalIndexKind::kInverted; std::vector files; // blob kinds only + // kInverted only: the dictionary carries at least one entry with a dropped posting list. + // Not serialized per entry; it raises kFeatureDroppedPostings on the directory. + bool dropped_postings = false; }; class MetadataDirectory { diff --git a/be/src/storage/index/snii/writer/logical_index_writer.cpp b/be/src/storage/index/snii/writer/logical_index_writer.cpp index a9683c132dc546..10c940c11ab194 100644 --- a/be/src/storage/index/snii/writer/logical_index_writer.cpp +++ b/be/src/storage/index/snii/writer/logical_index_writer.cpp @@ -196,6 +196,7 @@ class StreamingTermEncoder { entry->term = std::move(postings_->term); entry->df = total_docs_; entry->posting_dropped = true; + ++writer_->dropped_posting_terms_; *stats = stats_; return Status::OK(); } diff --git a/be/src/storage/index/snii/writer/logical_index_writer.h b/be/src/storage/index/snii/writer/logical_index_writer.h index 7e6e7875d76d6f..30247b76bdabcf 100644 --- a/be/src/storage/index/snii/writer/logical_index_writer.h +++ b/be/src/storage/index/snii/writer/logical_index_writer.h @@ -311,6 +311,9 @@ class LogicalIndexWriter { format::IndexTier tier() const { return tier_; } uint64_t index_id() const { return index_id_; } const std::string& index_suffix() const { return index_suffix_; } + // Dictionary entries written without a posting list (stop-gram). Non-zero raises + // kFeatureDroppedPostings on the container's directory. + uint64_t dropped_posting_terms() const { return dropped_posting_terms_; } // Builds the three mandatory v1 metadata blobs. The orchestrator writes them // contiguously as Core -> STI -> DBD and publishes their absolute references @@ -373,6 +376,7 @@ class LogicalIndexWriter { const std::vector& terms_; // materialized fallback (may be empty) SpimiTermBuffer* term_source_; // streaming source (null => use terms_) uint64_t term_count_ = 0; // distinct terms actually consumed + uint64_t dropped_posting_terms_ = 0; // of which posting_dropped (stop-gram) const std::vector& encoded_norms_; std::optional gram_scheme_; // See SniiIndexInput::stop_gram_df_threshold. 0 leaves every posting list in place. diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.cpp b/be/src/storage/index/snii/writer/snii_compound_writer.cpp index 89256f88dd7272..f5dcce744ab2ba 100644 --- a/be/src/storage/index/snii/writer/snii_compound_writer.cpp +++ b/be/src/storage/index/snii/writer/snii_compound_writer.cpp @@ -576,6 +576,9 @@ Status SniiCompoundWriter::write_tail() { LogicalIndexMetadataRef entry; entry.index_id = w.index_id(); entry.index_suffix = w.index_suffix(); + // Inherited groups above never carry this: only a positional index can be inherited + // (compaction/eligibility.cpp), and postings are dropped from docs-only ones alone. + entry.dropped_postings = w.dropped_posting_terms() > 0; entry.core_metadata = {.offset = out_->bytes_written(), .length = group.core.size()}; RETURN_IF_ERROR(append(group.core)); DORIS_CHECK_EQ(out_->bytes_written(), diff --git a/be/test/storage/index/snii/format/metadata_directory_test.cpp b/be/test/storage/index/snii/format/metadata_directory_test.cpp index be53aba0577e5c..895d62bf4af15f 100644 --- a/be/test/storage/index/snii/format/metadata_directory_test.cpp +++ b/be/test/storage/index/snii/format/metadata_directory_test.cpp @@ -452,5 +452,26 @@ TEST(SniiMetadataDirectory, RejectsNullOutputPointers) { encode_metadata_directory(sample_entries(), nullptr).is()); } +// An inverted entry whose dictionary dropped postings raises kFeatureDroppedPostings, so a +// reader that predates the flag refuses the container as unsupported instead of parsing a +// truncated locator; a directory without such an entry keeps its pre-feature bytes. +TEST(SniiMetadataDirectory, DroppedPostingsRaiseTheFeatureAndRoundTrip) { + auto entries = sample_entries(); + entries[0].dropped_postings = true; + const auto bytes = encode(entries); + doris::snii::SniiMetadataDirectoryPB wire; + ASSERT_TRUE(wire.ParseFromArray(bytes.data(), static_cast(bytes.size()))); + ASSERT_EQ(1, wire.required_features_size()); + EXPECT_EQ(kFeatureDroppedPostings, wire.required_features(0)); + MetadataDirectory directory; + ASSERT_TRUE(MetadataDirectory::decode(Slice(bytes), &directory).ok()); + EXPECT_EQ(directory.size(), 2U); + + const auto bare = encode(sample_entries()); + doris::snii::SniiMetadataDirectoryPB bare_wire; + ASSERT_TRUE(bare_wire.ParseFromArray(bare.data(), static_cast(bare.size()))); + EXPECT_EQ(0, bare_wire.required_features_size()); +} + } // namespace } // namespace doris::snii::format From 098d41d4278771313e0b7b18e0d0f69bbe2bfd34 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 11 Sep 2026 09:49:25 +0800 Subject: [PATCH 10/36] [test](be,regression) stop-gram suite proves a dropped entry reaches the query path; digest cost test drops its wall-clock gate test_gram_stop_gram repeated the same queries with the inverted-index result cache on, so the later phases could be served the first pass's bitmaps without touching the index, and a refused config update went unnoticed. The suite now turns the result cache off, asserts each BE config update succeeded, and runs a pattern whose grams straddle the line -- the dropped grams of the shared prefix and the kept grams of one rare marker -- checking in the profile that the index resolved the dictionary: a handful of candidate rows bounded by the kept posting, thousands of rows pruned, and no gate give-up. high_df_digest_cost_test asserted a five-sample wall-clock ratio, which scheduler preemption or a shared runner can fail on unchanged code. It now pins the deterministic invariant (the digest never exceeds kMaxHighDfDigestTerms entries) and keeps the timing diagnostic-only. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01METuP3aVRn8dfCU62PivnF --- .../snii/writer/high_df_digest_cost_test.cpp | 7 +-- .../gram/test_gram_stop_gram.groovy | 48 +++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/be/test/storage/index/snii/writer/high_df_digest_cost_test.cpp b/be/test/storage/index/snii/writer/high_df_digest_cost_test.cpp index 42a5cea3f6c195..c07c76c4103909 100644 --- a/be/test/storage/index/snii/writer/high_df_digest_cost_test.cpp +++ b/be/test/storage/index/snii/writer/high_df_digest_cost_test.cpp @@ -170,9 +170,10 @@ TEST(SniiHighDfDigestCost, BuildingTheDigestIsANegligibleShareOfTheWrite) { entries > 0 ? std::log2(static_cast(entries)) : 0.0); EXPECT_GT(entries, 0U) << "the corpus must actually populate the digest"; - EXPECT_LT(overhead_pct, 5.0) - << "digest build cost " << overhead_pct << "% of the write; it is meant to be " - << "one comparison per term plus a bounded heap insertion for the few that pass"; + // The gate is deterministic: the digest exists and stays bounded. The wall-clock share + // above is printed for a reader and never asserted -- five samples on a shared runner + // cannot separate a 5% effect from scheduler noise in either direction. + EXPECT_LE(entries, format::kMaxHighDfDigestTerms); } } // namespace doris::snii::writer diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy index f5afeb0e7a5c21..bca81ae6b9b10f 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +import org.apache.doris.regression.action.ProfileAction + +import java.util.regex.Pattern + // stop-gram end to end: an index built with the posting lists of very common grams dropped // must still answer LIKE/REGEXP exactly, and must actually have dropped them. // @@ -59,6 +63,10 @@ suite("test_gram_stop_gram", "p0") { def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) logger.info("update ${key}=${value}: code=${code}, out=${out}, err=${err}") + // A phase that runs against a config that never changed proves nothing. + assertEquals(0, code, "updating ${key} on backend ${backend_id} failed: ${err}") + assertTrue(out.toString().contains("OK"), + "updating ${key} on backend ${backend_id} was refused: ${out}") } } @@ -133,6 +141,10 @@ suite("test_gram_stop_gram", "p0") { try { sql "SET enable_sql_cache=false" sql "SET enable_condition_cache=false" + // The inverted-index result cache is keyed by the raw query bytes, so a second pass + // over the same queries could be served the first pass's bitmaps without touching + // the index at all; every phase below has to reach the index. + sql "SET enable_inverted_index_query_cache=false" // One batch keeps all 4000 rows in a single segment: above the row floor, dropping // is in effect. @@ -171,6 +183,42 @@ suite("test_gram_stop_gram", "p0") { "the single-segment index (${droppedBytes} bytes) should be far smaller than " + "the five-segment one (${keptBytes} bytes): its common postings were not dropped") + // A dropped entry has to reach the query path physically, not only through answers + // that a scan would give as well. This pattern's grams straddle the line: the ones of + // `common_text` sit in every row and were dropped, `rare_marker_1000` sits in one row + // and was kept. The AND resolves both in the dictionary, leaves the dropped ones out + // as match-all and reads the kept posting, so the profile must show a handful of + // candidates, thousands of rows pruned and no gate give-up. + def parseProfileCounter = { String profileString, String name -> + def exact = Pattern.compile(Pattern.quote(name) + ":\\s*[^\\(\\n]*\\((\\d+)\\)") + .matcher(profileString) + if (exact.find()) { + return Long.parseLong(exact.group(1)) + } + def plain = Pattern.compile(Pattern.quote(name) + ":\\s*(\\d+)").matcher(profileString) + assertTrue(plain.find(), "${name} is not parseable from profile") + return Long.parseLong(plain.group(1)) + } + def profileAction = new ProfileAction(context) + def gramCounters = ["RowsGramIndexFiltered", "GramIndexCandidateRows", "GramIndexGateGaveUp"] + sql "SET enable_inverted_index_query=true" + sql "SET enable_profile=true" + sql "SET profile_level=2" + def straddling = sql """/* gram_stop_gram_straddle */ SELECT COUNT(*) FROM ${oneSegment} + WHERE msg REGEXP 'common_text rare_marker_1000 '""" + def straddleProfile = profileAction.getProfileBySql("gram_stop_gram_straddle", gramCounters) + sql "SET enable_profile=false" + assertEquals(1L, straddling[0][0] as long, "exactly one row carries rare_marker_1000") + def candidates = parseProfileCounter(straddleProfile, "GramIndexCandidateRows") + def pruned = parseProfileCounter(straddleProfile, "RowsGramIndexFiltered") + def gaveUp = parseProfileCounter(straddleProfile, "GramIndexGateGaveUp") + logger.info("straddling pattern: candidates=${candidates} pruned=${pruned} gaveUp=${gaveUp}") + assertEquals(0L, gaveUp, "the retained gram keeps the node under budget; the gate must not give up") + assertTrue(candidates >= 1L && candidates <= 4L, + "the kept posting (df 4) bounds the candidates, got ${candidates}: the dropped " + + "grams must read as match-all and the kept one must still be read") + assertTrue(pruned >= rows - 4L, "the index pruned only ${pruned} of ${rows} rows") + // The gate could otherwise mask a broken dropped-gram path by giving up first, so // check again with its fallback ratio disabled. The gate's primary budget comes from // the segment itself and cannot be switched off -- that is the point of deriving it From 4d251a88c494bff2a3b26b445ed592b0902c5aed Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 11 Sep 2026 15:32:36 +0800 Subject: [PATCH 11/36] [test](be,regression) pin which LIKE escape forms the index accelerates, and drop the two queries that accelerate nothing `FunctionLikeBase::can_evaluate_inverted_index` accepts the three-child form of LIKE only with a backslash escape, which is the default escaping spelled out, so such a pattern is compiled and accelerated like a two-child one. A LIKE carrying its own escape character is refused: the gram compiler implements the default escaping only, so `%100!%%` ESCAPE `!`, which means "contains 100%", would be compiled as if `!` were an ordinary byte and would drop the row that matches. A unit test now pins both directions on the arguments alone. test_gram_regexp_like loses its two custom-escape patterns. They accelerated nothing -- a custom escape is exactly the case the index refuses -- so in a suite whose subject is index acceleration they only measured the fallback, while reaching a scan-node path unrelated to it: the function push-down that turns a LIKE into a LikeColumnPredicate accepts only a two-child call and asserts on the child count. That assertion is reported separately. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01METuP3aVRn8dfCU62PivnF --- .../exprs/function/like_gram_binding_test.cpp | 23 +++++++++++++++++++ .../gram/test_gram_regexp_like.out | 12 ---------- .../gram/test_gram_regexp_like.groovy | 7 ++---- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/be/test/exprs/function/like_gram_binding_test.cpp b/be/test/exprs/function/like_gram_binding_test.cpp index 25eac0d927c80b..f25da762beb4fb 100644 --- a/be/test/exprs/function/like_gram_binding_test.cpp +++ b/be/test/exprs/function/like_gram_binding_test.cpp @@ -28,6 +28,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/field.h" +#include "exprs/function/like.h" #include "exprs/vcompound_pred.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr_context.h" @@ -513,5 +514,27 @@ TEST_F(LikeGramBindingTest, MatchIsSkippedWhenTheCurrentSchemeDiffersFromTheSegm EXPECT_FALSE(bitmap->isEmpty()) << "the segment's own scheme answers the query"; } +// Which LIKE forms the index may answer. A backslash escape is the default escaping spelled out, +// so that three-child form is compiled and accelerated like the two-child one. A LIKE carrying its +// own escape character is not: the gram compiler implements the default escaping only, so +// `%100!%%` ESCAPE `!` -- which means "contains 100%" -- would be compiled as if `!` were an +// ordinary byte and would look for rows holding `100!`, dropping the row that matches. It is +// refused on its arguments, before any index is opened, and the predicate is evaluated on the rows. +TEST_F(LikeGramBindingTest, OnlyTheDefaultEscapingIsEligibleForTheIndex) { + const DataTypePtr string_type = std::make_shared(); + const auto literal = [&](const std::string& value) { + return VLiteral::create_shared(string_type, Field::create_field(value)); + }; + auto column = VSlotRef::create_shared(0, 0, 0, string_type, "p"); + auto pattern = literal("%100!%%"); + auto like = FunctionLike::create(); + + EXPECT_TRUE(like->can_evaluate_inverted_index({column, pattern})); + EXPECT_TRUE(like->can_evaluate_inverted_index({column, pattern, literal("\\")})); + EXPECT_FALSE(like->can_evaluate_inverted_index({column, pattern, literal("!")})); + // An escape that is not a literal cannot be read at all, so it is refused as well. + EXPECT_FALSE(like->can_evaluate_inverted_index({column, pattern, column})); +} + } // namespace } // namespace doris diff --git a/regression-test/data/inverted_index_p0/gram/test_gram_regexp_like.out b/regression-test/data/inverted_index_p0/gram/test_gram_regexp_like.out index 6a325ee2f3aa20..ebfeceaf6292e8 100644 --- a/regression-test/data/inverted_index_p0/gram/test_gram_regexp_like.out +++ b/regression-test/data/inverted_index_p0/gram/test_gram_regexp_like.out @@ -1140,12 +1140,6 @@ -- !compound_6_idx_true -- 1 --- !compound_7_idx_true -- -17 - --- !compound_8_idx_true -- -17 - -- !count_idx_true -- 5 @@ -2290,12 +2284,6 @@ -- !compound_6_idx_false -- 1 --- !compound_7_idx_false -- -17 - --- !compound_8_idx_false -- -17 - -- !count_idx_false -- 5 diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy index 6beca82b2b457f..625320ec980847 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy @@ -204,9 +204,8 @@ suite("test_gram_regexp_like", "p0") { '%CODE = UNAVAILABLE%', ] - // Compound predicates: an OR of REGEXPs, a REGEXP inside NOT (a AND b), REGEXP mixed with - // LIKE, and a LIKE with a custom ESCAPE (handled conservatively on the BE side: it is not - // pushed down, but it must still be correct) + // Compound predicates: an OR of REGEXPs, a REGEXP inside NOT (a AND b), and REGEXP mixed + // with LIKE def compounds = [ "msg REGEXP 'rpc' OR msg REGEXP '微博'", "NOT (msg REGEXP 'rpc' AND id > 5)", @@ -215,8 +214,6 @@ suite("test_gram_regexp_like", "p0") { "NOT (msg REGEXP 'error') OR msg REGEXP 'abc'", "NOT (msg LIKE '%rpc%' AND msg REGEXP 'error')", "msg REGEXP 'code' AND msg NOT LIKE '%Internal%'", - "msg LIKE '%100!%%' ESCAPE '!'", - "msg LIKE '%100!% do%' ESCAPE '!'", ] // Build the [tag, sql] list; the tag does not contain the raw pattern, which keeps the From c0d7588a1ee9b12117652edf8ba6df64eea1af4f Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 11 Sep 2026 18:14:47 +0800 Subject: [PATCH 12/36] [test](be,regression) drop the operand-order check, and stop the suites depending on a randomised variable or leaking their policies Three unrelated corrections to this PR's own test surface and one removal. The removal: `can_evaluate_inverted_index`. It was a hook on the function interface that let LIKE and REGEXP refuse arguments whose roles the index cannot answer for -- a pattern operand that is the indexed column, as in `'abc' LIKE col`, where the column holds patterns rather than values. It is gone, together with the two suite cases that exercised that shape, so a reversed call is now offered to the index like any other. Nothing else used the hook, and normal `col LIKE 'pattern'` never needed it. What did need attention is the assertions it used to justify: evaluate_gram_index asserted on the shape of its arguments because the hook had checked them. It now answers only the shape it can compile -- one indexed column and one constant pattern -- and returns without touching the index otherwise, so `LIKE ... ESCAPE`, which arrives as a second literal, is left to the rows instead of being compiled with the default escaping. test_gram_regexp_like and test_gram_metadata_inherit pin enable_function_pushdown to its default. With it on, the scan node turns a LIKE into a LikeColumnPredicate and, on a duplicate-key or merge-on-write table, removes the conjunct from the expression tree, so the gram index is never consulted and the profile assertions in both suites see nothing pruned. The pipeline randomises that variable per session, which is why they passed on one run and failed on the next. Four suites created index policies and dropped them only on their next run. Policies are instance-wide and the pipeline's cluster is shared, so every build left a dozen behind and eventually a suite that had nothing to do with gram indexes failed with "Index policy number cannot exceed 100". They now drop what they create. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01METuP3aVRn8dfCU62PivnF --- be/src/exprs/function/function.h | 14 +--- be/src/exprs/function/like.cpp | 39 +++------- be/src/exprs/function/like.h | 10 +-- be/src/exprs/vexpr.cpp | 4 -- .../exprs/function/like_gram_binding_test.cpp | 68 +++--------------- .../exprs/function/like_gram_index_test.cpp | 29 ++++---- .../inverted/gram/regex_gram_recall_test.cpp | 10 +-- .../gram/test_gram_pattern_recall.out | 72 ------------------- .../gram/test_gram_metadata_inherit.groovy | 13 ++++ .../gram/test_gram_pattern_recall.groovy | 13 ++-- .../gram/test_gram_policy_recovery.groovy | 6 ++ .../gram/test_gram_regexp_like.groovy | 13 ++++ 12 files changed, 80 insertions(+), 211 deletions(-) diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h index a19f56b602071e..784401efd43b6d 100644 --- a/be/src/exprs/function/function.h +++ b/be/src/exprs/function/function.h @@ -216,12 +216,6 @@ class IFunctionBase { } } - // Validate operand roles before VExpr separates indexed fields from literals. Functions - // whose index semantics depend on argument order must inspect the original children here. - virtual bool can_evaluate_inverted_index(const VExprSPtrs& /*function_arguments*/) const { - return true; - } - // True when this function's index push-down can only ever answer with an approximate // (superset) candidate set instead of the exact row set. Such a result is usable in one // place only -- narrowing the candidate rows of a conjunct that stays pushed down for @@ -229,8 +223,8 @@ class IFunctionBase { // computed and then dropped, rather than paying for the index reads first. virtual bool index_result_is_approximate() const { return false; } - // VExpr calls this only after can_evaluate_inverted_index accepts the ordered arguments - // and the indexed fields have been bound to storage-compatible types and iterators. + // VExpr calls this after binding the indexed fields to storage-compatible types and + // iterators; the arguments carry the call's literals, in child order. virtual Status evaluate_inverted_index( const ColumnsWithTypeAndName& arguments, const std::vector& data_type_with_names, @@ -522,10 +516,6 @@ class DefaultFunction final : public IFunctionBase { return function->close(context, scope); } - bool can_evaluate_inverted_index(const VExprSPtrs& function_arguments) const override { - return function->can_evaluate_inverted_index(function_arguments); - } - bool index_result_is_approximate() const override { return function->index_result_is_approximate(); } diff --git a/be/src/exprs/function/like.cpp b/be/src/exprs/function/like.cpp index 129ef848bfbf30..5252352622b480 100644 --- a/be/src/exprs/function/like.cpp +++ b/be/src/exprs/function/like.cpp @@ -1108,31 +1108,6 @@ Status FunctionRegexpLike::open(FunctionContext* context, return Status::OK(); } -bool FunctionLikeBase::can_evaluate_inverted_index(const VExprSPtrs& function_arguments) const { - if (function_arguments.size() != 2 && - !(get_name() == FunctionLike::name && function_arguments.size() == 3)) { - return false; - } - // VExpr subsequently validates that stripping this cast preserves the storage type. - if (!VExpr::expr_without_cast(function_arguments[0])->is_slot_ref() || - !function_arguments[1]->is_literal()) { - return false; - } - if (function_arguments.size() == 3) { - if (!function_arguments[2]->is_literal()) { - return false; - } - const auto* escape = assert_cast(function_arguments[2].get()); - Field escape_field; - escape->get_column_ptr()->get(0, escape_field); - // The gram compiler implements the default LIKE escaping semantics only. - if (escape_field.is_null() || escape_field.get() != "\\") { - return false; - } - } - return true; -} - // R8 (unity build): file-scope helpers use a namespace private to this file. namespace like_gram_index_detail { @@ -1185,11 +1160,15 @@ Status FunctionLikeBase::evaluate_gram_index( if (!config::enable_gram_index_regexp) { return Status::OK(); } - // Operand roles were accepted before VExpr separated literals and indexed fields. - DORIS_CHECK_EQ(iterators.size(), 1); - DORIS_CHECK_EQ(data_type_with_names.size(), 1); - DORIS_CHECK(!arguments.empty()); - DORIS_CHECK(is_column_const(*arguments[0].column)); + // VExpr binds whatever children it finds -- one entry per indexed column, one per literal + // -- so the shape reaching here is not guaranteed. Answer only the shape this compiler + // understands: a single indexed column and a single constant pattern. `LIKE ... ESCAPE` + // arrives with a second literal and would otherwise be compiled with the default escaping, + // which is not what the query asked for. + if (iterators.size() != 1 || data_type_with_names.size() != 1 || arguments.size() != 1 || + !is_column_const(*arguments[0].column)) { + return Status::OK(); + } Field pattern_field; arguments[0].column->get(0, pattern_field); if (pattern_field.is_null()) { diff --git a/be/src/exprs/function/like.h b/be/src/exprs/function/like.h index 8f521282f8bff6..078acf20c1258e 100644 --- a/be/src/exprs/function/like.h +++ b/be/src/exprs/function/like.h @@ -292,8 +292,6 @@ class FunctionLikeBase : public IFunction { Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t /*input_rows_count*/) const override; - bool can_evaluate_inverted_index(const VExprSPtrs& function_arguments) const override; - friend struct VectorAllpassSearchState; friend struct VectorEqualSearchState; friend struct VectorSubStringSearchState; @@ -395,11 +393,9 @@ class FunctionLikeBase : public IFunction { // Hard semantic constraint (Rulings R26 / R29): the index may only produce a superset of // candidates, and any index-side failure or inapplicable case may only cost the speedup -- // all of them simply return OK() without writing bitmap_result, and a problem on the index - // side must never make a LIKE/REGEXP query fail or change its result. The caller first - // validates the ordered children through can_evaluate_inverted_index: the value is the - // indexed operand, the pattern is a literal, and LIKE ESCAPE is absent or a literal - // backslash. VExpr then binds the single value iterator. This method handles a disabled - // switch, NULL pattern, unsupported index, compiler result of ALL and index errors. + // side must never make a LIKE/REGEXP query fail or change its result. This method handles + // a disabled switch, a call shape it cannot compile, a NULL pattern, an unsupported index, + // a compiler result of ALL and index errors. // The only statuses rethrown are CANCELLED / MEM_LIMIT_EXCEEDED / MEM_ALLOC_FAILED. enum class GramCompileKind { LIKE, REGEXP }; Status evaluate_gram_index(GramCompileKind kind, const ColumnsWithTypeAndName& arguments, diff --git a/be/src/exprs/vexpr.cpp b/be/src/exprs/vexpr.cpp index a036b230295d5d..46fa77adef29b6 100644 --- a/be/src/exprs/vexpr.cpp +++ b/be/src/exprs/vexpr.cpp @@ -919,10 +919,6 @@ ColumnPtr VExpr::get_result_from_const(size_t count) const { Status VExpr::_evaluate_inverted_index(VExprContext* context, const FunctionBasePtr& function, uint32_t segment_num_rows) { - if (!function->can_evaluate_inverted_index(children())) { - return Status::OK(); - } - // A function that can only answer approximately (the gram push-down behind LIKE / REGEXP) // produces a superset of candidate rows, and the caller names the single expression whose // superset it will read back. Anywhere else -- an operand of a compound AND/OR/NOT, a diff --git a/be/test/exprs/function/like_gram_binding_test.cpp b/be/test/exprs/function/like_gram_binding_test.cpp index f25da762beb4fb..745c4a5efb7ac6 100644 --- a/be/test/exprs/function/like_gram_binding_test.cpp +++ b/be/test/exprs/function/like_gram_binding_test.cpp @@ -158,8 +158,8 @@ class LikeGramBindingTest : public testing::Test { return index_file_writer.finish_close(); } - Status evaluate_pattern(const std::string& name, bool column_on_left, - const std::string& literal_value, PatternEvaluation* result) { + Status evaluate_pattern(const std::string& name, const std::string& literal_value, + PatternEvaluation* result) { const DataTypePtr string_type = std::make_shared(); TFunction function; TFunctionName function_name; @@ -179,13 +179,8 @@ class LikeGramBindingTest : public testing::Test { auto slot = VSlotRef::create_shared(0, 0, 0, string_type, "p"); auto literal = VLiteral::create_shared(string_type, Field::create_field(literal_value)); - if (column_on_left) { - expression->add_child(slot); - expression->add_child(literal); - } else { - expression->add_child(literal); - expression->add_child(slot); - } + expression->add_child(slot); + expression->add_child(literal); std::vector storage_types {{"p", string_type}}; std::unordered_map> index_status; @@ -330,19 +325,9 @@ class LikeGramBindingTest : public testing::Test { segment_v2::IndexQueryContextPtr _query_context; }; -TEST_F(LikeGramBindingTest, LikePreservesOperandRolesAndPrunesIndexedValues) { - PatternEvaluation reverse; - auto status = evaluate_pattern("like", /*column_on_left=*/false, "abcdef", &reverse); - ASSERT_TRUE(status.ok()) << status; - ASSERT_TRUE(reverse.row_result->get_bool(0)); // 'abcdef' LIKE '%' - ASSERT_TRUE(reverse.row_result->get_bool(1)); - ASSERT_FALSE(reverse.row_result->get_bool(2)); - EXPECT_FALSE(reverse.has_exact_result); - EXPECT_TRUE(reverse.candidates.is_empty() || reverse.candidates.get_data_bitmap()->contains(0)) - << "The index on the pattern operand must not discard a TRUE LIKE row"; - +TEST_F(LikeGramBindingTest, LikePrunesIndexedValues) { PatternEvaluation forward; - status = evaluate_pattern("like", /*column_on_left=*/true, "%abcdef%", &forward); + auto status = evaluate_pattern("like", "%abcdef%", &forward); ASSERT_TRUE(status.ok()) << status; check_selective_result(forward); } @@ -363,25 +348,14 @@ TEST_F(LikeGramBindingTest, LikeUsesTheWrittenSchemeAfterSameNamePolicyReplaceme _policy_mgr.apply_policy_changes({tokenizer, analyzer}, {6753803, 6753802}); PatternEvaluation result; - const auto status = evaluate_pattern("like", /*column_on_left=*/true, "%abcdef%", &result); + const auto status = evaluate_pattern("like", "%abcdef%", &result); ASSERT_TRUE(status.ok()) << status; check_selective_result(result); } -TEST_F(LikeGramBindingTest, RegexpPreservesOperandRolesAndPrunesIndexedValues) { - PatternEvaluation reverse; - auto status = evaluate_pattern("regexp", /*column_on_left=*/false, "abcdef%", &reverse); - ASSERT_TRUE(status.ok()) << status; - ASSERT_TRUE(reverse.row_result->get_bool(0)); // 'abcdef%' REGEXP '%' - ASSERT_TRUE(reverse.row_result->get_bool(1)); - ASSERT_FALSE(reverse.row_result->get_bool(2)); - EXPECT_FALSE(reverse.has_exact_result); - EXPECT_TRUE(reverse.candidates.is_empty() || reverse.candidates.get_data_bitmap()->contains(0)) - << "The index on the pattern operand must not discard a TRUE REGEXP row"; - EXPECT_TRUE(reverse.candidates.is_empty() || reverse.candidates.get_data_bitmap()->contains(1)); - +TEST_F(LikeGramBindingTest, RegexpPrunesIndexedValues) { PatternEvaluation forward; - status = evaluate_pattern("regexp", /*column_on_left=*/true, "abcdef", &forward); + auto status = evaluate_pattern("regexp", "abcdef", &forward); ASSERT_TRUE(status.ok()) << status; check_selective_result(forward); } @@ -436,7 +410,7 @@ TEST_F(LikeGramBindingTest, PatternsUseTheSelectedReaderSchemeInASharedContainer {"like", "%abcdef%"}, {"regexp", "abcdef"}}) { SCOPED_TRACE(name); PatternEvaluation result; - status = evaluate_pattern(name, /*column_on_left=*/true, pattern, &result); + status = evaluate_pattern(name, pattern, &result); ASSERT_TRUE(status.ok()) << status; check_selective_result(result); } @@ -514,27 +488,5 @@ TEST_F(LikeGramBindingTest, MatchIsSkippedWhenTheCurrentSchemeDiffersFromTheSegm EXPECT_FALSE(bitmap->isEmpty()) << "the segment's own scheme answers the query"; } -// Which LIKE forms the index may answer. A backslash escape is the default escaping spelled out, -// so that three-child form is compiled and accelerated like the two-child one. A LIKE carrying its -// own escape character is not: the gram compiler implements the default escaping only, so -// `%100!%%` ESCAPE `!` -- which means "contains 100%" -- would be compiled as if `!` were an -// ordinary byte and would look for rows holding `100!`, dropping the row that matches. It is -// refused on its arguments, before any index is opened, and the predicate is evaluated on the rows. -TEST_F(LikeGramBindingTest, OnlyTheDefaultEscapingIsEligibleForTheIndex) { - const DataTypePtr string_type = std::make_shared(); - const auto literal = [&](const std::string& value) { - return VLiteral::create_shared(string_type, Field::create_field(value)); - }; - auto column = VSlotRef::create_shared(0, 0, 0, string_type, "p"); - auto pattern = literal("%100!%%"); - auto like = FunctionLike::create(); - - EXPECT_TRUE(like->can_evaluate_inverted_index({column, pattern})); - EXPECT_TRUE(like->can_evaluate_inverted_index({column, pattern, literal("\\")})); - EXPECT_FALSE(like->can_evaluate_inverted_index({column, pattern, literal("!")})); - // An escape that is not a literal cannot be read at all, so it is refused as well. - EXPECT_FALSE(like->can_evaluate_inverted_index({column, pattern, column})); -} - } // namespace } // namespace doris diff --git a/be/test/exprs/function/like_gram_index_test.cpp b/be/test/exprs/function/like_gram_index_test.cpp index 95d9e474ee402c..333ec6e1a9e590 100644 --- a/be/test/exprs/function/like_gram_index_test.cpp +++ b/be/test/exprs/function/like_gram_index_test.cpp @@ -91,22 +91,21 @@ TEST(LikeGramIndexTest, SuccessfulIndexResultIsApproximate) { } } -TEST(LikeGramIndexTest, CustomEscapeIsNotPushedDown) { +TEST(LikeGramIndexTest, AnEscapeArgumentIsNotPushedDown) { + // `LIKE ... ESCAPE` arrives as two literals, the pattern and the escape character. The gram + // compiler implements the default escaping only, so compiling the pattern here would read + // that character as an ordinary byte and answer a different question. The call is left to + // the rows instead, and the index is not read at all. FunctionLike function; - const DataTypePtr type = std::make_shared(); - auto value = VSlotRef::create_shared(0, 0, 0, type, "value"); - auto pattern = VLiteral::create_shared(type, Field::create_field("%abcd%")); - auto custom_escape = VLiteral::create_shared(type, Field::create_field("#")); - auto default_escape = VLiteral::create_shared(type, Field::create_field("\\")); - auto variable_escape = VSlotRef::create_shared(1, 1, 1, type, "escape"); - - // Eligibility is checked while every operand still has its original position, including - // an ESCAPE column that has no index and would disappear from the flattened arguments. - EXPECT_FALSE(function.can_evaluate_inverted_index({value, pattern, custom_escape})); - EXPECT_FALSE(function.can_evaluate_inverted_index({value, pattern, variable_escape})); - EXPECT_FALSE( - function.can_evaluate_inverted_index({value, pattern, default_escape, custom_escape})); - EXPECT_TRUE(function.can_evaluate_inverted_index({value, pattern, default_escape})); + ColumnsWithTypeAndName args {const_string_arg("%100!%%"), const_string_arg("!")}; + std::vector names {{"msg", std::make_shared()}}; + RecordingGramIndexIterator iterator; + segment_v2::InvertedIndexResultBitmap result; + const auto status = + function.evaluate_inverted_index(args, names, {&iterator}, 100, nullptr, result); + ASSERT_TRUE(status.ok()) << status; + EXPECT_FALSE(iterator.queried) << "a call this compiler cannot answer must not read the index"; + EXPECT_TRUE(result.is_empty()); } TEST(LikeGramIndexTest, ConfigDisabledSkipsPushDown) { diff --git a/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp b/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp index 574b79ac3f8ff2..bd036dd6b3480c 100644 --- a/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp +++ b/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp @@ -149,13 +149,6 @@ Status scalar_matches(const std::string& pattern, const std::vector context->set_constant_cols(constant_columns); Function function; - if (dynamic_pattern) { - // Dynamic-pattern execution is a scalar compatibility boundary, not an index - // acceleration claim: the eligibility hook rejects a pattern column. - auto value = VSlotRef::create_shared(0, 0, 0, string_type, "value"); - auto pattern_slot = VSlotRef::create_shared(1, 1, 1, string_type, "pattern"); - EXPECT_FALSE(function.can_evaluate_inverted_index({value, pattern_slot})); - } RETURN_IF_ERROR(function.open(context.get(), FunctionContext::THREAD_LOCAL)); const auto* state = static_cast(context->get_function_state(FunctionContext::THREAD_LOCAL)); @@ -630,10 +623,9 @@ TEST(RegexGramRecallTest, InvalidPatternsRejectAcrossEngineOptions) { } } -TEST(RegexGramRecallTest, DynamicPatternRe2ExecutionAndIndexEligibility) { +TEST(RegexGramRecallTest, DynamicPatternRe2Execution) { // A ColumnString pattern reaches vector_non_const -> scalar_function -> RE2 even // with Hyperscan fallback disabled. Extended regex does not enable Boost on this path. - // Each execution also checks that can_evaluate_inverted_index rejects the pattern slot. const std::vector regexp_cases = { {R"(ab\vcdtimeout)", {"ab\vcdtimeout", "ab\ncdtimeout", "ab\rcdtimeout", "unrelated"}, diff --git a/regression-test/data/inverted_index_p0/gram/test_gram_pattern_recall.out b/regression-test/data/inverted_index_p0/gram/test_gram_pattern_recall.out index 01de562ad49086..d327fb02e46a41 100644 --- a/regression-test/data/inverted_index_p0/gram/test_gram_pattern_recall.out +++ b/regression-test/data/inverted_index_p0/gram/test_gram_pattern_recall.out @@ -183,15 +183,6 @@ -- !dense_false_nul_like_2 -- 28 --- !dense_false_reverse_like -- -32 - --- !dense_false_reverse_regexp -- -12 -19 -30 -33 - -- !dense_false_null -- 31 @@ -384,15 +375,6 @@ -- !sparse_false_nul_like_2 -- 28 --- !sparse_false_reverse_like -- -32 - --- !sparse_false_reverse_regexp -- -12 -19 -30 -33 - -- !sparse_false_null -- 31 @@ -585,15 +567,6 @@ -- !dense_lc_false_nul_like_2 -- 28 --- !dense_lc_false_reverse_like -- -32 - --- !dense_lc_false_reverse_regexp -- -12 -19 -30 -33 - -- !dense_lc_false_null -- 31 @@ -786,15 +759,6 @@ -- !sparse_lc_false_nul_like_2 -- 28 --- !sparse_lc_false_reverse_like -- -32 - --- !sparse_lc_false_reverse_regexp -- -12 -19 -30 -33 - -- !sparse_lc_false_null -- 31 @@ -987,15 +951,6 @@ -- !dense_true_nul_like_2 -- 28 --- !dense_true_reverse_like -- -32 - --- !dense_true_reverse_regexp -- -12 -19 -30 -33 - -- !dense_true_null -- 31 @@ -1188,15 +1143,6 @@ -- !sparse_true_nul_like_2 -- 28 --- !sparse_true_reverse_like -- -32 - --- !sparse_true_reverse_regexp -- -12 -19 -30 -33 - -- !sparse_true_null -- 31 @@ -1389,15 +1335,6 @@ -- !dense_lc_true_nul_like_2 -- 28 --- !dense_lc_true_reverse_like -- -32 - --- !dense_lc_true_reverse_regexp -- -12 -19 -30 -33 - -- !dense_lc_true_null -- 31 @@ -1590,15 +1527,6 @@ -- !sparse_lc_true_nul_like_2 -- 28 --- !sparse_lc_true_reverse_like -- -32 - --- !sparse_lc_true_reverse_regexp -- -12 -19 -30 -33 - -- !sparse_lc_true_null -- 31 diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy index 322753075672cd..423abe2017496a 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy @@ -81,6 +81,11 @@ suite("test_gram_metadata_inherit", "p0") { // it, so both passes would read one and the same filter result and this parity check would // degenerate into a tautology. sql "SET enable_condition_cache=false" + // With the scan node's function push-down on, a LIKE becomes a storage-layer + // LikeColumnPredicate and never reaches the gram index, so the profile assertion below + // would see nothing pruned. The pipeline randomises this variable per session, so pin + // it to its default. + sql "SET enable_function_pushdown=false" def checkPatterns = { String phase -> [false, true].each { useIndex -> @@ -150,4 +155,12 @@ suite("test_gram_metadata_inherit", "p0") { FROM test_gram_metadata_inherit WHERE tag MATCH 'fox'""" order_qt_inherited_and_new """SELECT /*+ SET_VAR(enable_match_without_inverted_index=false) */ id FROM test_gram_metadata_inherit WHERE dense LIKE '%abcdefgh%' AND tag MATCH 'fox'""" + // The policies live cluster-wide and the cluster is shared, so a suite that leaves + // its own behind eats into the instance-wide policy limit for everyone else. + sql "DROP TABLE IF EXISTS test_gram_metadata_inherit" + ['dense', 'sparse'].each { name -> + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_inherit_${name}" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_inherit_${name}_tok" + } + } diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy index 3cba5eb313bb0d..d333868e4cb2a5 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy @@ -117,10 +117,6 @@ suite("test_gram_pattern_recall", "p0") { "order_qt_${column}_${useIndex}_nul_like_${i}"("""SELECT id FROM test_gram_pattern_recall WHERE ${column} LIKE ${sqlLiteral(pattern)}""") } - "order_qt_${column}_${useIndex}_reverse_like"("""SELECT id - FROM test_gram_pattern_recall WHERE 'abctimeout' LIKE ${column}""") - "order_qt_${column}_${useIndex}_reverse_regexp"("""SELECT id - FROM test_gram_pattern_recall WHERE 'abctimeout' REGEXP ${column}""") "order_qt_${column}_${useIndex}_null"("""SELECT id FROM test_gram_pattern_recall WHERE ${column} IS NULL""") "order_qt_${column}_${useIndex}_compound"("""SELECT id @@ -129,4 +125,13 @@ suite("test_gram_pattern_recall", "p0") { } } sql "SET enable_inverted_index_query=true" + // The policies live cluster-wide and the cluster is shared, so a suite that leaves + // its own behind eats into the instance-wide policy limit for everyone else. + sql "DROP TABLE IF EXISTS test_gram_pattern_recall" + schemes.each { name, properties -> + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_recall_${name}" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_recall_${name}_tok" + } + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_recall_invalid_tokenizer" + } diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy index 18ba0bc50cc806..0d60dad82ae8c6 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy @@ -122,4 +122,10 @@ suite("test_gram_policy_recovery", "p0") { WHERE msg MATCH_ALL 'abcdef'""") } sql "SET enable_inverted_index_query=true" + // The policies live cluster-wide and the cluster is shared, so a suite that leaves + // its own behind eats into the instance-wide policy limit for everyone else. + sql "DROP TABLE IF EXISTS test_gram_policy_recovery" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_recovery_analyzer" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_recovery_tokenizer" + } diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy index 625320ec980847..5b7e2d753c915f 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy @@ -258,6 +258,11 @@ suite("test_gram_regexp_like", "p0") { // it, so both passes would read one and the same filter result and this parity check would // degenerate into a tautology. sql "SET enable_condition_cache=false" + // Pin the scan node's function push-down to its default. With it on, a LIKE is turned + // into a storage-layer LikeColumnPredicate and never reaches the gram index, so the + // profile assertions below would see nothing pruned. The pipeline randomises this + // variable per session, which would make those assertions pass or fail by luck. + sql "SET enable_function_pushdown=false" // Produce the .out golden: run the same batch of queries once with the index on and once off def runAll = { boolean useIndex -> @@ -367,4 +372,12 @@ suite("test_gram_regexp_like", "p0") { runParityCheck("after_delete") sql "SET enable_inverted_index_query=true" + // The policies live cluster-wide and the cluster is shared, so a suite that leaves + // its own behind eats into the instance-wide policy limit for everyone else. + sql "DROP TABLE IF EXISTS ${tbl}" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS ${sparseAna}" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS ${denseLcAna}" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS ${sparseTok}" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS ${denseLcTok}" + } From f62da7db81008f4f890f508e1c026b717287f1c3 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 11 Sep 2026 19:46:00 +0800 Subject: [PATCH 13/36] [fix](fe) validate an omitted max_gram against the default the backend applies The validator substituted 16 for an absent max_gram and did not persist it, while GramScheme defaults the same absent property to 4. A tokenizer created with only mode=sparse and min_gram=5 therefore passed DDL validation, the table was created, and the first load failed on the backend with max_gram(4) < min_gram(5) -- a table that looked created and could not be written. Both defaults now come from named constants that mirror GramScheme, so a definition the writer cannot honour is refused where it is written. --- .../indexpolicy/NGramTokenizerValidator.java | 13 +++++- .../indexpolicy/GramDdlValidationTest.java | 44 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NGramTokenizerValidator.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NGramTokenizerValidator.java index d7d8ea6dd1a05a..d8fa3e074d4b19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NGramTokenizerValidator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NGramTokenizerValidator.java @@ -56,6 +56,10 @@ public class NGramTokenizerValidator extends BasePolicyValidator { private static final int MIN_GRAM_UPPER_BOUND = 64; private static final int MAX_GRAM_LOWER_BOUND = 1; private static final int MAX_GRAM_UPPER_BOUND = 256; + // What GramScheme applies when max_gram is absent. It has to match BE's member initializer: + // assuming a larger one here lets DDL through that BE then refuses at the first load. + private static final int GRAM_MAX_GRAM_DEFAULT = 4; + private static final int GRAM_MIN_GRAM_DEFAULT = 3; private static final double MIN_DENSITY = 0.001; // Persist a decimal spelling that BE can parse verbatim: Java-only type suffixes or // silently trimmed trailing whitespace must not pass DDL validation. @@ -216,8 +220,13 @@ private void validateGramMode(Map props, String mode) throws Ddl throw new DdlException("ngram tokenizer mode must be one of " + VALID_MODES + ", got: '" + mode + "'" + (mode.isEmpty() ? " (empty)" : "")); } - int minGram = parseIntInRange(props, "min_gram", 3, MIN_GRAM_LOWER_BOUND, MIN_GRAM_UPPER_BOUND); - int maxGram = parseIntInRange(props, "max_gram", 16, MAX_GRAM_LOWER_BOUND, MAX_GRAM_UPPER_BOUND); + int minGram = parseIntInRange(props, "min_gram", GRAM_MIN_GRAM_DEFAULT, + MIN_GRAM_LOWER_BOUND, MIN_GRAM_UPPER_BOUND); + // The default must be the one GramScheme applies when the property is absent: FE used + // to assume 16, so "mode=sparse, min_gram=5" passed here and then failed at the first + // load with max_gram(4) < min_gram(5). Reject it while the user is still writing DDL. + int maxGram = parseIntInRange(props, "max_gram", GRAM_MAX_GRAM_DEFAULT, + MAX_GRAM_LOWER_BOUND, MAX_GRAM_UPPER_BOUND); if (minGram > maxGram) { throw new DdlException("min_gram (" + minGram + ") must be <= max_gram (" + maxGram + ")"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/GramDdlValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/GramDdlValidationTest.java index 05620b6def8fe7..cdaf5a3d7efef3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/GramDdlValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/GramDdlValidationTest.java @@ -83,6 +83,50 @@ public void testResolveGramMode() { // ------------- validateAnalyzerGraphLocked (R31, triggered via createIndexPolicy) ------------- + @Test + public void testOmittedMaxGramIsValidatedAgainstTheDefaultBackendApplies() { + // GramScheme defaults an absent max_gram to 4. FE used to assume 16 here, so this DDL + // was accepted and the first load then failed on the backend with + // "max_gram(4) < min_gram(5)" -- a table that looked created but could not be written. + Map tooLong = new HashMap<>(); + tooLong.put("type", "ngram"); + tooLong.put("mode", "sparse"); + tooLong.put("min_gram", "5"); + UserException e = Assertions.assertThrows(UserException.class, + () -> manager.createIndexPolicy(false, "omitted_max_tok", + IndexPolicyTypeEnum.TOKENIZER, tooLong)); + Assertions.assertTrue(e.getMessage().contains("min_gram (5) must be <= max_gram (4)"), + e.getMessage()); + + // Spelling out a max_gram the min fits under is still accepted, and so is a min_gram + // that fits under the default. This harness has no EditLog, so a definition that passes + // validation goes on to fail in the persist step; reaching that step is the proof. + Map spelled = new HashMap<>(); + spelled.put("type", "ngram"); + spelled.put("mode", "sparse"); + spelled.put("min_gram", "5"); + spelled.put("max_gram", "8"); + expectValidationAccepts("spelled_max_tok", spelled); + + Map withinDefault = new HashMap<>(); + withinDefault.put("type", "ngram"); + withinDefault.put("mode", "sparse"); + withinDefault.put("min_gram", "3"); + expectValidationAccepts("within_default_tok", withinDefault); + } + + // Validation accepted the tokenizer iff createIndexPolicy got past it; without an EditLog + // the persist step then throws, and that throw is the signal. + private void expectValidationAccepts(String name, Map props) { + try { + manager.createIndexPolicy(false, name, IndexPolicyTypeEnum.TOKENIZER, props); + } catch (UserException e) { + Assertions.fail("validation rejected " + name + ": " + e.getMessage()); + } catch (RuntimeException expectedWithoutEditLog) { + // Reaching the persist step is the proof that validation accepted the tokenizer. + } + } + @Test public void testLowercaseFilterRejectedWithSparseMode() { Map props = new HashMap<>(); From 001e556751b2c421d9e9a7f67c0983952b916cb0 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 11 Sep 2026 19:46:00 +0800 Subject: [PATCH 14/36] [fix](be) gram compiler: refuse ill-formed UTF-8, and stop growing a query past the budget Two independent defects in compilation. The code-point decoder accepted overlong, surrogate and out-of-range sequences and handed back the code point they spell. A regexp literal holding the bytes C0 AF then compiled to the grams of "/", which a row holding those same bytes never stored, because the extractor treats them as a separator. Where the extended-regex fallback matches the original bytes, intersecting that bitmap drops a true row before the scalar recheck. Such a sequence now decodes to a non-code-point sentinel and stays a separator, exactly as every other invalid byte does. The gram budget was applied to the finished query. Each additional byte of a long high-entropy literal was therefore merged, deduplicated and sorted into a vector that kept growing, and only afterwards was the whole thing collapsed to ALL: quadratic work per segment for a result that is discarded. GramQuery::and_ and or_ now answer ALL as soon as a merged result passes the leaf cap, so neither operand ever exceeds it and each step is bounded. The compiler's own constant aliases the cap so the two cannot drift apart. --- .../index/inverted/gram/gram_query.cpp | 6 ++++ .../storage/index/inverted/gram/gram_query.h | 6 ++++ .../storage/index/inverted/gram/regex_ast.cpp | 13 +++++++ .../index/inverted/gram/regex_gram_compiler.h | 2 +- .../index/inverted/gram/gram_query_test.cpp | 34 +++++++++++++++++++ .../gram/regex_gram_compiler_test.cpp | 31 +++++++++++++++++ 6 files changed, 91 insertions(+), 1 deletion(-) diff --git a/be/src/storage/index/inverted/gram/gram_query.cpp b/be/src/storage/index/inverted/gram/gram_query.cpp index 55d349fd80ccc4..8ca2f95c12aad7 100644 --- a/be/src/storage/index/inverted/gram/gram_query.cpp +++ b/be/src/storage/index/inverted/gram/gram_query.cpp @@ -154,6 +154,9 @@ GramQuery GramQuery::and_(GramQuery a, GramQuery b) { if (r.grams.empty() && r.subs.size() == 1) { return std::move(r.subs[0]); } + if (r.leaf_count() > kMaxLeaves) { + return all(); + } return r; } @@ -213,6 +216,9 @@ GramQuery GramQuery::or_(GramQuery a, GramQuery b) { if (r.grams.empty() && r.subs.size() == 1) { return std::move(r.subs[0]); } + if (r.leaf_count() > kMaxLeaves) { + return all(); + } return r; } diff --git a/be/src/storage/index/inverted/gram/gram_query.h b/be/src/storage/index/inverted/gram/gram_query.h index f92c3af5ffb901..a1c8654640d946 100644 --- a/be/src/storage/index/inverted/gram/gram_query.h +++ b/be/src/storage/index/inverted/gram/gram_query.h @@ -57,6 +57,12 @@ struct GramQuery { // and collapse single-element nodes. static GramQuery and_(GramQuery a, GramQuery b); static GramQuery or_(GramQuery a, GramQuery b); + // The largest query worth building. A compiler discards anything past this and reads no + // index, so a combination that would cross it is answered with ALL right away rather than + // grown, deduplicated and sorted first: on a long high-entropy pattern that work is + // quadratic in the pattern's length and its result is thrown away. ALL is the conservative + // answer everywhere -- dropping constraints only widens the candidate set. + static constexpr size_t kMaxLeaves = 64; bool is_all() const { return op == Op::ALL; } bool is_none() const { return op == Op::NONE; } // Total number of gram leaves, counted recursively over all sub-queries. diff --git a/be/src/storage/index/inverted/gram/regex_ast.cpp b/be/src/storage/index/inverted/gram/regex_ast.cpp index 1ee85f0ae72018..458e92943a53ee 100644 --- a/be/src/storage/index/inverted/gram/regex_ast.cpp +++ b/be/src/storage/index/inverted/gram/regex_ast.cpp @@ -104,6 +104,19 @@ uint32_t decode_one_cp(std::string_view s, size_t* consumed) { } v = (v << 6) | (cc & 0x3FU); } + // A sequence can be well-formed byte by byte and still be ill-formed as UTF-8. Decoding one + // to the code point it spells would be worse than dropping it, because the compiler would + // then demand grams of that code point's canonical encoding -- bytes the row never held. The + // extractor treats these bytes as a separator and stores no gram across them, so a row + // holding `C0 AF` stores nothing for `/`, while `C0 AF` decoded as U+002F asks for `/` + // grams and would filter that row away. Three shapes are rejected here: + // - overlong: fewer bits set than the length promises (`C0 AF` for U+002F); + // - surrogate halves U+D800..U+DFFF, which UTF-8 may not encode; + // - anything above U+10FFFF. + static constexpr uint32_t kOverlongFloor[5] = {0, 0, 0x80, 0x800, 0x10000}; + if (v < kOverlongFloor[l] || (v >= 0xD800U && v <= 0xDFFFU) || v > kMaxCodePoint) { + return 0x110000U + c; + } *consumed = static_cast(l); return v; } diff --git a/be/src/storage/index/inverted/gram/regex_gram_compiler.h b/be/src/storage/index/inverted/gram/regex_gram_compiler.h index df50ad2a43998b..9e3907e569ef54 100644 --- a/be/src/storage/index/inverted/gram/regex_gram_compiler.h +++ b/be/src/storage/index/inverted/gram/regex_gram_compiler.h @@ -83,7 +83,7 @@ class RegexGramCompiler { // request, so such a query can spend far more on index IO than the scan it is supposed to // save. Over the budget the query degrades to ALL and the scan runs unaided -- giving up the // acceleration, never the correctness. - static constexpr size_t kMaxQueryGrams = 64; + static constexpr size_t kMaxQueryGrams = GramQuery::kMaxLeaves; private: // Degrade *q to ALL when its gram leaves exceed kMaxQueryGrams. Applied to the result of both diff --git a/be/test/storage/index/inverted/gram/gram_query_test.cpp b/be/test/storage/index/inverted/gram/gram_query_test.cpp index 79628e84d5106a..729d40522f14e7 100644 --- a/be/test/storage/index/inverted/gram/gram_query_test.cpp +++ b/be/test/storage/index/inverted/gram/gram_query_test.cpp @@ -74,4 +74,38 @@ TEST(GramQueryTest, StructuralKeyDistinguishesGramBytesFromOperators) { GramQuery::of_gram("a").structural_key()); } +// A combination that crosses the size a compiler would discard is answered with ALL instead of +// being grown, deduplicated and sorted first. Without this, compiling a long high-entropy +// pattern was quadratic in its length and threw the result away: 4,096 characters cost seconds +// of CPU per segment for a query that was already ALL by character 256. +TEST(GramQueryTest, CombiningStopsOnceThePlanIsPastTheSizeAnyoneWouldUse) { + GramQuery wide = GramQuery::of_gram("aaa"); + for (size_t i = 0; i < GramQuery::kMaxLeaves + 8; ++i) { + std::string gram = "b"; + gram += static_cast('a' + (i % 26)); + gram += static_cast('a' + (i / 26)); + wide = GramQuery::and_(std::move(wide), GramQuery::of_gram(gram)); + EXPECT_LE(wide.leaf_count(), GramQuery::kMaxLeaves) + << "an AND grew past the cap at step " << i; + if (wide.is_all()) { + break; + } + } + EXPECT_TRUE(wide.is_all()) << "the AND should have collapsed once it crossed the cap"; + + GramQuery alternatives = GramQuery::none(); + for (size_t i = 0; i < GramQuery::kMaxLeaves + 8; ++i) { + std::string gram = "c"; + gram += static_cast('a' + (i % 26)); + gram += static_cast('a' + (i / 26)); + alternatives = GramQuery::or_(std::move(alternatives), GramQuery::of_gram(gram)); + EXPECT_LE(alternatives.leaf_count(), GramQuery::kMaxLeaves) + << "an OR grew past the cap at step " << i; + if (alternatives.is_all()) { + break; + } + } + EXPECT_TRUE(alternatives.is_all()) << "the OR should have collapsed once it crossed the cap"; +} + } // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/regex_gram_compiler_test.cpp b/be/test/storage/index/inverted/gram/regex_gram_compiler_test.cpp index 2c995efdbf33e4..5256de6813657e 100644 --- a/be/test/storage/index/inverted/gram/regex_gram_compiler_test.cpp +++ b/be/test/storage/index/inverted/gram/regex_gram_compiler_test.cpp @@ -156,4 +156,35 @@ TEST(RegexGramCompilerTest, LikeEscapeConservative) { EXPECT_TRUE(q.is_all()); } +// A pattern is compiled into grams the index is then asked for, so a byte sequence that no +// encoder emits must not be turned into one that does. `C0 AF` spells U+002F the way an overlong +// encoding does; decoding it would ask the index for grams of `/`, while a row holding those raw +// bytes stores no gram across them at all (the extractor stops at a non-ASCII byte), so the row +// would be filtered away. The same holds for a surrogate half and for a value above U+10FFFF. +TEST(RegexGramCompilerTest, IllFormedUtf8IsNotDecodedIntoTheCodePointItSpells) { + const std::string slash = dense("abc/def"); + EXPECT_EQ(slash, R"(("/de" & "abc" & "bc/" & "c/d" & "def"))"); + + EXPECT_NE(dense("abc\xC0\xAF" + "def"), + slash) + << "overlong C0 AF must not become '/'"; + EXPECT_NE(dense("abc\xE0\x80\xAF" + "def"), + slash) + << "overlong E0 80 AF must not become '/'"; + // A surrogate half and an out-of-range value are ill-formed too; what matters is that each + // stays a separator, so the literal on either side is all the pattern can promise. + EXPECT_EQ(dense("abc\xED\xA0\x80" + "def"), + R"(("abc" & "def"))"); + EXPECT_EQ(dense("abc\xF5\x80\x80\x80" + "def"), + R"(("abc" & "def"))"); + // The ASCII runs on either side still constrain the query. + EXPECT_EQ(dense("abc\xC0\xAF" + "def"), + R"(("abc" & "def"))"); +} + } // namespace doris::segment_v2::gram From 434df14e3a5046000f99078902066fbe6a94710e Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 11 Sep 2026 19:46:01 +0800 Subject: [PATCH 15/36] [fix](be) gram scheme fence covers MATCH_REGEXP and a segment with no persisted scheme MATCH_REGEXP carries a raw pattern, but its answer is still decided by analyzer output: the scalar function matches the pattern against the terms the current analyzer cuts each row into, while the index matches it against the persisted dictionary. On a dense-3 segment read under a dense-4 analyzer those are different sets, so the index returned an empty exact bitmap where the scalar predicate matched. It now joins the analyzed subtypes, which also keeps it out of the scheme-blind result cache. The fence itself tested the segment's scheme for presence rather than comparing the two optionals. A segment written by a legacy ngram tokenizer carries no scheme at all and its dictionary holds that tokenizer's own terms, so after the policy is recreated in gram mode and the table recovered, MATCH looked current grams up in that dictionary and answered nothing. An absent persisted scheme is now a mismatch like any other. --- .../storage/index/snii/snii_index_reader.cpp | 18 +++-- .../gram/test_gram_policy_recovery.out | 42 ++++++++++++ .../gram/test_gram_policy_recovery.groovy | 65 ++++++++++++++++++- 3 files changed, 117 insertions(+), 8 deletions(-) diff --git a/be/src/storage/index/snii/snii_index_reader.cpp b/be/src/storage/index/snii/snii_index_reader.cpp index 1122a95090e206..bf5edb08d9a8fb 100644 --- a/be/src/storage/index/snii/snii_index_reader.cpp +++ b/be/src/storage/index/snii/snii_index_reader.cpp @@ -165,10 +165,13 @@ bool uses_phrase_frequency_scoring(InvertedIndexQueryType query_type, query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY); } -// Query types whose terms come out of the current analyzer. On a gram-family index those -// terms are grams, and they only mean what the segment's own grams mean when both were cut -// by the same scheme; a gram query compiles against the segment's scheme itself and a raw -// pattern query never analyzes, so neither is affected. +// Query types whose answer is decided by terms the current analyzer produced. On a +// gram-family index those terms are grams, and they only mean what the segment's own grams +// mean when both were cut by the same scheme. MATCH_REGEXP belongs here even though its +// pattern is raw: the scalar function matches that pattern against the terms the current +// analyzer cuts each row into, while the index matches it against the persisted dictionary, +// so the two answer the same question only when both were cut alike. A gram query compiles +// against the segment's own scheme, so it is not affected. bool analyzes_query_terms(InvertedIndexQueryType query_type) { switch (query_type) { case InvertedIndexQueryType::MATCH_ANY_QUERY: @@ -176,6 +179,7 @@ bool analyzes_query_terms(InvertedIndexQueryType query_type) { case InvertedIndexQueryType::MATCH_PHRASE_QUERY: case InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY: case InvertedIndexQueryType::MATCH_PHRASE_EDGE_QUERY: + case InvertedIndexQueryType::MATCH_REGEXP_QUERY: return true; default: return false; @@ -767,7 +771,11 @@ Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::s const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, &logical_reader)); - if (analyzed_query && logical_reader->gram_scheme().has_value() && + // Compare the two optionals, not just two schemes: a segment written by a legacy ngram + // tokenizer carries no scheme at all, and its dictionary holds that tokenizer's terms. Once + // the current analyzer cuts grams, looking those grams up in that dictionary answers a + // different question, so an absent persisted scheme is a mismatch like any other. + if (analyzed_query && current_gram_scheme.has_value() && current_gram_scheme != logical_reader->gram_scheme()) { return Status::Error( "gram index segment was cut with a different scheme than the current " diff --git a/regression-test/data/inverted_index_p0/gram/test_gram_policy_recovery.out b/regression-test/data/inverted_index_p0/gram/test_gram_policy_recovery.out index f8628e28138b27..18d54f0e781ae5 100644 --- a/regression-test/data/inverted_index_p0/gram/test_gram_policy_recovery.out +++ b/regression-test/data/inverted_index_p0/gram/test_gram_policy_recovery.out @@ -21,6 +21,10 @@ 1 2 +-- !recovered_false_match_regexp -- +1 +2 + -- !recovered_true_like -- 1 2 @@ -37,6 +41,10 @@ 1 2 +-- !recovered_true_match_regexp -- +1 +2 + -- !mixed_false_like -- 1 2 @@ -69,6 +77,11 @@ 2 5 +-- !mixed_false_match_regexp -- +1 +2 +5 + -- !mixed_true_like -- 1 2 @@ -101,3 +114,32 @@ 2 5 +-- !mixed_true_match_regexp -- +1 +2 +5 + +-- !legacy_false_match_any -- +1 +2 + +-- !legacy_false_match_all -- +1 +2 + +-- !legacy_false_like -- +1 +2 + +-- !legacy_true_match_any -- +1 +2 + +-- !legacy_true_match_all -- +1 +2 + +-- !legacy_true_like -- +1 +2 + diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy index 0d60dad82ae8c6..3763a38fa9f4c3 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy @@ -25,13 +25,13 @@ suite("test_gram_policy_recovery", "p0") { // degenerate into a tautology. sql "SET enable_condition_cache=false" - def waitAnalyzerInstalled = { int expectedTokenCount -> + def waitAnalyzerInstalledNamed = { String name, int expectedTokenCount -> def deadline = System.currentTimeMillis() + 180_000 Exception lastNotFound = null while (System.currentTimeMillis() < deadline) { try { def result = sql """SELECT JSON_LENGTH(TOKENIZE('abcdef', - '"analyzer"="gram_recovery_analyzer"'))""" + '"analyzer"="${name}"'))""" // The replacement has the same name: successful lookup alone could still // observe the old policy before BE receives the next heartbeat update. if (result[0][0].toString() == expectedTokenCount.toString()) { @@ -46,9 +46,12 @@ suite("test_gram_policy_recovery", "p0") { sleep(1000) } throw new IllegalStateException( - "gram_recovery_analyzer did not produce ${expectedTokenCount} tokens on BE", + "${name} did not produce ${expectedTokenCount} tokens on BE", lastNotFound) } + def waitAnalyzerInstalled = { int expectedTokenCount -> + waitAnalyzerInstalledNamed("gram_recovery_analyzer", expectedTokenCount) + } sql "DROP TABLE IF EXISTS test_gram_policy_recovery" sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_recovery_analyzer" @@ -96,6 +99,12 @@ suite("test_gram_policy_recovery", "p0") { WHERE msg MATCH_ANY 'abcdef'""") "order_qt_recovered_${useIndex}_match_all"("""SELECT id FROM test_gram_policy_recovery WHERE msg MATCH_ALL 'abcdef'""") + // MATCH_REGEXP carries a raw pattern, but it is matched against terms: the scalar + // function against what the current analyzer cuts each row into, the index against the + // dictionary the segment was written with. On a dense3 segment read under a dense4 + // analyzer those are different sets, so the index has to step aside here as well. + "order_qt_recovered_${useIndex}_match_regexp"("""SELECT id FROM test_gram_policy_recovery + WHERE msg MATCH_REGEXP '^abcd\$'""") } // This writer sees dense4, while the recovered rowset retains its dense3 dictionary. @@ -120,8 +129,58 @@ suite("test_gram_policy_recovery", "p0") { WHERE msg MATCH_ANY 'abcdef'""") "order_qt_mixed_${useIndex}_match_all"("""SELECT id FROM test_gram_policy_recovery WHERE msg MATCH_ALL 'abcdef'""") + "order_qt_mixed_${useIndex}_match_regexp"("""SELECT id FROM test_gram_policy_recovery + WHERE msg MATCH_REGEXP '^abcd\$'""") + } + + // A second recovery, this time from a tokenizer that predates gram mode. Such a segment + // records no gram scheme, and its dictionary holds that tokenizer's own terms -- here + // bigrams. Once the policy is recreated in gram mode, an analyzed query is cut into grams + // that mean nothing in that dictionary, so the index has to step aside exactly as it does + // for a scheme that differs. "No scheme at all" is a mismatch, not an exemption. + sql "DROP TABLE IF EXISTS test_gram_legacy_recovery" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_legacy_analyzer" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_legacy_tokenizer" + sql """CREATE INVERTED INDEX TOKENIZER gram_legacy_tokenizer PROPERTIES ( + "type"="ngram", "min_gram"="2", "max_gram"="2")""" + sql """CREATE INVERTED INDEX ANALYZER gram_legacy_analyzer + PROPERTIES ("tokenizer"="gram_legacy_tokenizer")""" + waitAnalyzerInstalledNamed("gram_legacy_analyzer", 5) + + sql """CREATE TABLE test_gram_legacy_recovery ( + id INT, + msg VARCHAR(128), + INDEX idx_msg (msg) USING INVERTED PROPERTIES ("analyzer"="gram_legacy_analyzer") + ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true", + "inverted_index_storage_format"="SNII")""" + sql """INSERT INTO test_gram_legacy_recovery VALUES + (1, 'abcdef'), (2, 'old abcdef row'), (3, 'unrelated'), (4, NULL)""" + sql "sync" + + sql "DROP TABLE test_gram_legacy_recovery" + sql "DROP INVERTED INDEX ANALYZER gram_legacy_analyzer" + sql "DROP INVERTED INDEX TOKENIZER gram_legacy_tokenizer" + sql """CREATE INVERTED INDEX TOKENIZER gram_legacy_tokenizer PROPERTIES ( + "type"="ngram", "mode"="dense", "min_gram"="3")""" + sql """CREATE INVERTED INDEX ANALYZER gram_legacy_analyzer + PROPERTIES ("tokenizer"="gram_legacy_tokenizer")""" + waitAnalyzerInstalledNamed("gram_legacy_analyzer", 4) + sql "RECOVER TABLE test_gram_legacy_recovery" + + [false, true].each { useIndex -> + sql "SET enable_inverted_index_query=${useIndex}" + "order_qt_legacy_${useIndex}_match_any"("""SELECT id FROM test_gram_legacy_recovery + WHERE msg MATCH_ANY 'abcdef'""") + "order_qt_legacy_${useIndex}_match_all"("""SELECT id FROM test_gram_legacy_recovery + WHERE msg MATCH_ALL 'abcdef'""") + "order_qt_legacy_${useIndex}_like"("""SELECT id FROM test_gram_legacy_recovery + WHERE msg LIKE '%abcdef%'""") } sql "SET enable_inverted_index_query=true" + sql "DROP TABLE IF EXISTS test_gram_legacy_recovery" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_legacy_analyzer" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_legacy_tokenizer" // The policies live cluster-wide and the cluster is shared, so a suite that leaves // its own behind eats into the instance-wide policy limit for everyone else. sql "DROP TABLE IF EXISTS test_gram_policy_recovery" From ab1fccb8ab34737637b94558729e1439d96e39db Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 11 Sep 2026 19:46:01 +0800 Subject: [PATCH 16/36] [fix](be) record dropped postings per logical index, and charge the digest to the searcher cache BUILD INDEX can carry an unchanged gram index into a replacement container, copying its dictionary byte for byte -- locator-less stop-gram entries included. The rewritten container has to declare the dropped-posting feature again for it, or a reader without the feature parses those entries instead of refusing the container. It did not: the decoder never set the per-entry flag at all, so every entry read back said false, and the inheritance snapshot carried no such state. The fact is now recorded per logical index rather than only as a directory-wide feature. Keeping only the feature is too coarse for a rewrite: it would make a container that inherits solely the indexes that dropped nothing declare it anyway, fencing an older reader off a container it could read in full. The field is written only when true, so a directory that drops nothing keeps its previous bytes, and decode requires the per-entry flags and the feature word to agree in both directions. Separately, the searcher-cache charge left out the high-df digest, which is decoded with the core metadata and stays resident for the reader's whole life -- up to about 48 KiB per cached segment index. It is now added with the same saturating arithmetic as the norms charge. --- .../index/snii/format/metadata_directory.cpp | 24 ++++- .../index/snii/format/metadata_directory.h | 5 +- .../snii/reader/logical_index_reader.cpp | 9 ++ .../index/snii/reader/snii_segment_reader.cpp | 1 + .../index/snii/reader/snii_segment_reader.h | 7 ++ .../snii/writer/snii_compound_writer.cpp | 9 +- .../index/snii/writer/snii_compound_writer.h | 1 + .../snii/format/metadata_directory_test.cpp | 47 ++++++++++ .../snii_compound_writer_inherit_test.cpp | 91 +++++++++++++++++++ .../index/snii/writer/stop_gram_test.cpp | 49 ++++++++-- gensrc/proto/snii.proto | 8 ++ .../gram/test_gram_metadata_inherit.groovy | 65 +++++++++++++ 12 files changed, 303 insertions(+), 13 deletions(-) diff --git a/be/src/storage/index/snii/format/metadata_directory.cpp b/be/src/storage/index/snii/format/metadata_directory.cpp index 75d061dea9615c..303fe9f6fb2984 100644 --- a/be/src/storage/index/snii/format/metadata_directory.cpp +++ b/be/src/storage/index/snii/format/metadata_directory.cpp @@ -119,12 +119,16 @@ Status decode_directory_pb(const doris::snii::SniiMetadataDirectoryPB& input, // Row 0a/0b: whitelist of known required features; anything else is a // format this binary does not understand. bool feature_blob = false; + // Raised directory-wide as the fence for readers that cannot parse a shortened entry. Which + // index actually needs it is recorded per entry; the two must agree (checked below). + bool dropped_postings_declared = false; for (const uint32_t feature : input.required_features()) { if (feature == kFeatureBlobLogicalIndex) { feature_blob = true; continue; } if (feature == kFeatureDroppedPostings) { + dropped_postings_declared = true; // Understood by this reader's dictionary decoder; nothing to validate here. continue; } @@ -133,6 +137,7 @@ Status decode_directory_pb(const doris::snii::SniiMetadataDirectoryPB& input, } bool has_blob_entry = false; + bool has_dropped_entry = false; std::vector entries; entries.reserve(input.indexes_size()); for (const auto& index : input.indexes()) { @@ -151,6 +156,8 @@ Status decode_directory_pb(const doris::snii::SniiMetadataDirectoryPB& input, switch (kind_value) { case static_cast(LogicalIndexKind::kInverted): RETURN_IF_ERROR(decode_inverted_entry(index, &entry)); + entry.dropped_postings = index.dropped_postings(); + has_dropped_entry |= entry.dropped_postings; break; case static_cast(LogicalIndexKind::kBkd): case static_cast(LogicalIndexKind::kAnn): @@ -181,6 +188,13 @@ Status decode_directory_pb(const doris::snii::SniiMetadataDirectoryPB& input, return metadata_directory_corrupted( "metadata directory: blob feature flag disagrees with entries"); } + // Same argument for dropped postings. A shortened entry without the flag would be read by + // an old binary as a slim POD reference and reported as truncation; the flag without such + // an entry would fence that binary off a container it could read in full. + if (has_dropped_entry != dropped_postings_declared) { + return metadata_directory_corrupted( + "metadata directory: dropped-posting feature flag disagrees with entries"); + } *out = std::move(entries); return Status::OK(); } @@ -246,14 +260,20 @@ Status encode_metadata_directory(const std::vector& ent encode_blob_ref(entry.core_metadata, index->mutable_core_metadata()); encode_blob_ref(entry.sampled_term_index, index->mutable_sampled_term_index()); encode_blob_ref(entry.dict_block_directory, index->mutable_dict_block_directory()); - any_dropped = any_dropped || entry.dropped_postings; + // BYTE GATE again: field 8 is written only when the index really dropped a + // posting, so a directory that drops nothing keeps its pre-feature bytes. + if (entry.dropped_postings) { + index->set_dropped_postings(true); + any_dropped = true; + } } else { // The three inverted refs do not serialize for blob entries, so // the shared decode self-check below could not catch a caller that // filled them; reject that in-memory shape here. if (entry.core_metadata.offset != 0 || entry.core_metadata.length != 0 || entry.sampled_term_index.offset != 0 || entry.sampled_term_index.length != 0 || - entry.dict_block_directory.offset != 0 || entry.dict_block_directory.length != 0) { + entry.dict_block_directory.offset != 0 || entry.dict_block_directory.length != 0 || + entry.dropped_postings) { return metadata_directory_corrupted( "metadata directory: blob entry carries inverted metadata"); } diff --git a/be/src/storage/index/snii/format/metadata_directory.h b/be/src/storage/index/snii/format/metadata_directory.h index c012e887091dee..7c9ff89747d339 100644 --- a/be/src/storage/index/snii/format/metadata_directory.h +++ b/be/src/storage/index/snii/format/metadata_directory.h @@ -65,8 +65,9 @@ struct LogicalIndexMetadataRef { // original members so existing designated-initializer sites stay valid. LogicalIndexKind kind = LogicalIndexKind::kInverted; std::vector files; // blob kinds only - // kInverted only: the dictionary carries at least one entry with a dropped posting list. - // Not serialized per entry; it raises kFeatureDroppedPostings on the directory. + // kInverted only: this index's dictionary carries at least one entry with a dropped + // posting list. Serialized per entry, and it also raises kFeatureDroppedPostings on the + // directory as the fence for readers that cannot parse such an entry. bool dropped_postings = false; }; diff --git a/be/src/storage/index/snii/reader/logical_index_reader.cpp b/be/src/storage/index/snii/reader/logical_index_reader.cpp index 5ce803da145bcf..42405f1ee2340e 100644 --- a/be/src/storage/index/snii/reader/logical_index_reader.cpp +++ b/be/src/storage/index/snii/reader/logical_index_reader.cpp @@ -491,6 +491,15 @@ size_t LogicalIndexReader::memory_usage() const { return std::numeric_limits::max(); } bytes += norms_reserved_charge_; + // The high-df digest is decoded with the core metadata and stays resident for as long as + // this reader does. sizeof(*this) covers only the two vector headers, so add what they + // hold: at the writer's cap that is 4,096 hashes and frequencies, about 48 KiB. + const size_t digest = core_.high_df_terms.term_hash.capacity() * sizeof(uint64_t) + + core_.high_df_terms.df.capacity() * sizeof(uint32_t); + if (digest > std::numeric_limits::max() - bytes) { + return std::numeric_limits::max(); + } + bytes += digest; return bytes; } diff --git a/be/src/storage/index/snii/reader/snii_segment_reader.cpp b/be/src/storage/index/snii/reader/snii_segment_reader.cpp index da28d2245024fd..ea4c0573739410 100644 --- a/be/src/storage/index/snii/reader/snii_segment_reader.cpp +++ b/be/src/storage/index/snii/reader/snii_segment_reader.cpp @@ -239,6 +239,7 @@ Status SniiSegmentReader::load_inherited_index(const LogicalIndexKey& key, // Safe to sum and narrow: open() ran validate_metadata_group on every entry. out->index_id = entry->index_id; out->index_suffix = entry->index_suffix; + out->dropped_postings = entry->dropped_postings; out->core_length = static_cast(entry->core_metadata.length); out->sampled_term_index_length = static_cast(entry->sampled_term_index.length); out->dict_block_directory_length = static_cast(entry->dict_block_directory.length); diff --git a/be/src/storage/index/snii/reader/snii_segment_reader.h b/be/src/storage/index/snii/reader/snii_segment_reader.h index f0ab723015576f..d86f49e5a77f0b 100644 --- a/be/src/storage/index/snii/reader/snii_segment_reader.h +++ b/be/src/storage/index/snii/reader/snii_segment_reader.h @@ -67,6 +67,13 @@ struct InheritedLogicalIndex { size_t core_length = 0; size_t sampled_term_index_length = 0; size_t dict_block_directory_length = 0; + // Whether THIS index's dictionary holds an entry whose posting list was dropped. The + // dictionary is copied byte for byte, locator-less entries included, so the rewritten + // container has to declare kFeatureDroppedPostings for it again -- otherwise a reader + // without the feature parses those entries instead of refusing the container. Per index, + // not per container: a rewrite that keeps only the indexes that dropped nothing must not + // declare the feature at all. + bool dropped_postings = false; }; // Immutable, fully validated view of one container as an inheritance source for a diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.cpp b/be/src/storage/index/snii/writer/snii_compound_writer.cpp index f5dcce744ab2ba..92d817d4d0529c 100644 --- a/be/src/storage/index/snii/writer/snii_compound_writer.cpp +++ b/be/src/storage/index/snii/writer/snii_compound_writer.cpp @@ -142,6 +142,7 @@ Status SniiCompoundWriter::inherit(const reader::SniiRewriteSnapshot& snapshot, DORIS_CHECK_EQ(group.metadata_group.size(), group.core_length + group.sampled_term_index_length + group.dict_block_directory_length); + group.dropped_postings = index.dropped_postings; inherited_.push_back(std::move(group)); } return Status::OK(); @@ -555,6 +556,10 @@ Status SniiCompoundWriter::write_tail() { entry.dict_block_directory = { .offset = core_offset + group.core_length + group.sampled_term_index_length, .length = group.dict_block_directory_length}; + // The dictionary is copied byte for byte, locator-less stop-gram entries included, so + // this entry re-declares what the source index declared. Without it a reader that + // predates the feature would parse those entries instead of refusing the container. + entry.dropped_postings = group.dropped_postings; RETURN_IF_ERROR(append(group.metadata_group)); DORIS_CHECK_EQ(out_->bytes_written(), core_offset + group.metadata_group.size()); directory_entries.push_back(std::move(entry)); @@ -576,8 +581,8 @@ Status SniiCompoundWriter::write_tail() { LogicalIndexMetadataRef entry; entry.index_id = w.index_id(); entry.index_suffix = w.index_suffix(); - // Inherited groups above never carry this: only a positional index can be inherited - // (compaction/eligibility.cpp), and postings are dropped from docs-only ones alone. + // Inherited groups are handled above: they carry the source container's declaration, + // because their dictionaries are copied rather than written here. entry.dropped_postings = w.dropped_posting_terms() > 0; entry.core_metadata = {.offset = out_->bytes_written(), .length = group.core.size()}; RETURN_IF_ERROR(append(group.core)); diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.h b/be/src/storage/index/snii/writer/snii_compound_writer.h index cb3092d780db81..573a952a6df071 100644 --- a/be/src/storage/index/snii/writer/snii_compound_writer.h +++ b/be/src/storage/index/snii/writer/snii_compound_writer.h @@ -256,6 +256,7 @@ class SniiCompoundWriter { size_t core_length = 0; size_t sampled_term_index_length = 0; size_t dict_block_directory_length = 0; + bool dropped_postings = false; }; // One registered blob logical index awaiting finish(). cold/hot refs are diff --git a/be/test/storage/index/snii/format/metadata_directory_test.cpp b/be/test/storage/index/snii/format/metadata_directory_test.cpp index 895d62bf4af15f..de0acfa167374d 100644 --- a/be/test/storage/index/snii/format/metadata_directory_test.cpp +++ b/be/test/storage/index/snii/format/metadata_directory_test.cpp @@ -473,5 +473,52 @@ TEST(SniiMetadataDirectory, DroppedPostingsRaiseTheFeatureAndRoundTrip) { EXPECT_EQ(0, bare_wire.required_features_size()); } +// A rewrite copies an inherited dictionary byte for byte and has to declare the feature again +// for exactly the indexes it carried over. It therefore needs to know WHICH index dropped a +// posting, so the fact travels per entry and survives a round trip -- the directory-wide +// feature is the old-reader fence, not the record of who needed it. +TEST(SniiMetadataDirectory, ADecodedEntryReportsItsOwnDroppedPostingState) { + auto entries = sample_entries(); + entries[0].dropped_postings = true; + MetadataDirectory directory; + ASSERT_TRUE(MetadataDirectory::decode(Slice(encode(entries)), &directory).ok()); + ASSERT_EQ(directory.size(), 2U); + EXPECT_TRUE(directory.entries()[0].dropped_postings); + EXPECT_FALSE(directory.entries()[1].dropped_postings) + << "an index that dropped nothing must not inherit its neighbour's declaration"; + + MetadataDirectory bare; + ASSERT_TRUE(MetadataDirectory::decode(Slice(encode(sample_entries())), &bare).ok()); + for (const auto& entry : bare.entries()) { + EXPECT_FALSE(entry.dropped_postings); + } +} + +// The per-entry bit and the directory-wide feature have to agree in both directions, for the +// same reason the blob feature does: a shortened entry without the feature is read by an old +// binary as a slim POD reference and reported as truncation, while the feature without such an +// entry fences that binary off a container it could have read in full. +TEST(SniiMetadataDirectory, TheDroppedPostingFeatureAndTheEntriesMustAgree) { + auto entries = sample_entries(); + entries[0].dropped_postings = true; + const auto bytes = encode(entries); + + doris::snii::SniiMetadataDirectoryPB no_feature; + ASSERT_TRUE(no_feature.ParseFromArray(bytes.data(), static_cast(bytes.size()))); + no_feature.clear_required_features(); + std::string payload; + ASSERT_TRUE(no_feature.SerializeToString(&payload)); + MetadataDirectory decoded; + EXPECT_TRUE(MetadataDirectory::decode(Slice(payload), &decoded) + .is()); + + doris::snii::SniiMetadataDirectoryPB no_entry; + ASSERT_TRUE(no_entry.ParseFromArray(bytes.data(), static_cast(bytes.size()))); + no_entry.mutable_indexes(0)->clear_dropped_postings(); + ASSERT_TRUE(no_entry.SerializeToString(&payload)); + EXPECT_TRUE(MetadataDirectory::decode(Slice(payload), &decoded) + .is()); +} + } // namespace } // namespace doris::snii::format diff --git a/be/test/storage/index/snii/writer/snii_compound_writer_inherit_test.cpp b/be/test/storage/index/snii/writer/snii_compound_writer_inherit_test.cpp index d977b5fd2aaf98..997f3dcd0610d1 100644 --- a/be/test/storage/index/snii/writer/snii_compound_writer_inherit_test.cpp +++ b/be/test/storage/index/snii/writer/snii_compound_writer_inherit_test.cpp @@ -30,6 +30,9 @@ // and any read or write failure leaves no sealed container behind. // 5. The copy runs through a fixed-size buffer, so its memory does not grow // with the source file. +// 6. A declaration the copied bytes depend on travels with them: an inherited gram index +// whose dictionary holds locator-less shortened entries keeps its own dropped-posting +// declaration, and an index appended beside it does not acquire one. #include @@ -41,8 +44,10 @@ #include #include "common/status.h" +#include "storage/index/inverted/gram/gram_scheme.h" #include "storage/index/snii/format/bootstrap_header.h" #include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/metadata_directory.h" #include "storage/index/snii/format/tail_pointer.h" #include "storage/index/snii/io/file_reader.h" #include "storage/index/snii/io/file_writer.h" @@ -253,6 +258,48 @@ std::vector QueryTerm(const std::vector& image, uint64_t inde return docids; } +// Decodes an image's own metadata directory, which is where each entry's dropped-posting +// declaration lives. +MetadataDirectory ReadDirectory(const std::vector& image) { + const TailPointer tail = ReadTail(image); + MetadataDirectory directory; + EXPECT_LE(tail.directory_offset + tail.directory_length, image.size()); + EXPECT_TRUE(MetadataDirectory::decode(Slice(image.data() + tail.directory_offset, + static_cast(tail.directory_length)), + &directory) + .ok()); + return directory; +} + +// The only entry for (index_id, suffix) in a decoded directory. +const LogicalIndexMetadataRef& EntryOf(const MetadataDirectory& directory, uint64_t index_id, + const char* suffix) { + for (const auto& entry : directory.entries()) { + if (entry.index_id == index_id && entry.index_suffix == suffix) { + return entry; + } + } + ADD_FAILURE() << "no entry for index " << index_id << " suffix " << suffix; + static const LogicalIndexMetadataRef kMissing {}; + return kMissing; +} + +// A gram index whose commonest term is over the stop-gram threshold, so the writer keeps the +// dictionary entry and drops its posting. That shortened entry is what an old reader misparses +// and what the dropped-posting feature exists to fence off. +SniiIndexInput MakeStoppedGramInput(uint64_t index_id, const char* suffix, uint32_t doc_count) { + SniiIndexInput in = MakeInput(index_id, suffix, SingleTermCorpus(kTermA, doc_count), doc_count); + in.config = IndexConfig::kDocsOnly; + in.stop_gram_df_threshold = 1; + doris::segment_v2::gram::GramScheme scheme; + scheme.mode = doris::segment_v2::gram::GramMode::SPARSE; + scheme.min_len = 3; + scheme.max_len = 4; + scheme.density_permille = 250; + in.gram_scheme = scheme; + return in; +} + } // namespace TEST(SniiCompoundWriterInherit, CarriesInheritedIndexesAndAppendsANewOne) { @@ -281,6 +328,50 @@ TEST(SniiCompoundWriterInherit, CarriesInheritedIndexesAndAppendsANewOne) { EXPECT_EQ(expected_docids, QueryTerm(output, kIndexIdC, kSuffixC, kTermC)); } +// BUILD INDEX can hand an unchanged gram index to inherit_keys. Its dictionary is copied +// verbatim -- locator-less shortened entries included -- so the replacement container has to go +// on declaring dropped postings for THAT index. Without it, a reader that expects every entry to +// carry a locator reads the copied bytes as a slim POD reference and reports the container +// truncated, which is precisely the misparse the feature exists to prevent. The declaration is +// per index rather than per container for the other direction: an index appended beside it that +// dropped nothing must not acquire one, or a rewrite keeping only such indexes would fence an +// old reader off a container it could read in full. +TEST(SniiCompoundWriterInherit, InheritedIndexKeepsTheDroppedPostingDeclaration) { + // A production segment only drops postings above kHighDfDigestDivisor rows, so the source + // here is that size: below it the writer keeps every posting and the test would pass for + // the wrong reason. + const uint32_t stopped_doc_count = static_cast(kHighDfDigestDivisor); + SniiIndexInput stopped = MakeStoppedGramInput(kIndexIdA, kSuffixA, stopped_doc_count); + const std::vector source = WriteContainer({&stopped}); + + const MetadataDirectory source_directory = ReadDirectory(source); + ASSERT_TRUE(EntryOf(source_directory, kIndexIdA, kSuffixA).dropped_postings) + << "the source must really drop a posting, or this test proves nothing"; + + ImageFileReader source_reader(source); + SniiSegmentReader source_segment; + ASSERT_TRUE(SniiSegmentReader::open(&source_reader, &source_segment).ok()); + SniiRewriteSnapshot snapshot; + ASSERT_TRUE(source_segment + .prepare_rewrite_snapshot(KeysOf({{kIndexIdA, kSuffixA}}), + stopped_doc_count, &snapshot) + .ok()); + + MemoryFileWriter sink; + SniiCompoundWriter compound(&sink); + ASSERT_TRUE(compound.inherit(snapshot, &source_reader).ok()); + SniiIndexInput fresh = + MakeInput(kIndexIdC, kSuffixC, SingleTermCorpus(kTermC, kDocCount), kDocCount); + ASSERT_TRUE(compound.add_logical_index(fresh).ok()); + ASSERT_TRUE(compound.finish().ok()); + + const MetadataDirectory output_directory = ReadDirectory(sink.bytes()); + EXPECT_TRUE(EntryOf(output_directory, kIndexIdA, kSuffixA).dropped_postings) + << "the inherited entry lost the declaration its copied bytes depend on"; + // The index appended alongside it drops nothing, and inheritance must not claim it does. + EXPECT_FALSE(EntryOf(output_directory, kIndexIdC, kSuffixC).dropped_postings); +} + TEST(SniiCompoundWriterInherit, CopiesThePhysicalPrefixByteForByte) { const std::vector source = BuildSourceImage(); ImageFileReader source_reader(source); diff --git a/be/test/storage/index/snii/writer/stop_gram_test.cpp b/be/test/storage/index/snii/writer/stop_gram_test.cpp index 6dd76e7ce5a134..70d1b726f0313b 100644 --- a/be/test/storage/index/snii/writer/stop_gram_test.cpp +++ b/be/test/storage/index/snii/writer/stop_gram_test.cpp @@ -77,19 +77,21 @@ struct Segment { // gated on it, because a match-all term is only sound where the candidates get re-checked // and the digest is only ever read by the gram query's cost gate. void Build(Segment* out, const std::vector>& term_strides, - uint32_t threshold) { + uint32_t threshold, bool with_gram_scheme = true) { SniiIndexInput input; input.index_id = 11; input.index_suffix = "body"; input.config = format::IndexConfig::kDocsOnly; input.doc_count = kDocCount; input.stop_gram_df_threshold = threshold; - segment_v2::gram::GramScheme scheme; - scheme.mode = segment_v2::gram::GramMode::SPARSE; - scheme.min_len = 3; - scheme.max_len = 4; - scheme.density_permille = 250; - input.gram_scheme = scheme; + if (with_gram_scheme) { + segment_v2::gram::GramScheme scheme; + scheme.mode = segment_v2::gram::GramMode::SPARSE; + scheme.min_len = 3; + scheme.max_len = 4; + scheme.density_permille = 250; + input.gram_scheme = scheme; + } for (const auto& [term, stride] : term_strides) { input.terms.push_back(make_term(term, EveryNth(stride))); } @@ -240,6 +242,39 @@ TEST(SniiStopGram, HighDfDigestStaysProportionalToTheVocabulary) { EXPECT_GT(digest.df_ceiling, 0U) << "a truncated digest must still bound what it omits"; } +// The digest is decoded with the core metadata and stays resident for as long as the reader +// does, so the searcher cache has to be charged for it. Left out, the cache believes it holds +// less than it does -- by up to 48 KiB per cached segment index at the writer's cap, across +// however many segments the cache holds. +// +// Two readers over the same corpus isolate the charge: the only resident metadata the gram +// scheme adds is the digest, so the difference between them is what the digest costs. +TEST(SniiStopGram, TheSearcherCacheChargeIncludesTheResidentDigest) { + std::vector> corpus; + for (uint32_t i = 0; i < 300; i++) { + corpus.emplace_back("term_" + std::to_string(i), 2 + (i % 50)); + } + Segment with_digest; + Build(&with_digest, corpus, /*threshold=*/0); + Segment without_digest; + Build(&without_digest, corpus, /*threshold=*/0, /*with_gram_scheme=*/false); + + const auto& digest = with_digest.index.high_df_terms(); + ASSERT_FALSE(digest.empty()) + << "this corpus must reach the digest for the test to mean anything"; + ASSERT_TRUE(without_digest.index.high_df_terms().empty()); + + const size_t digest_bytes = digest.term_hash.capacity() * sizeof(uint64_t) + + digest.df.capacity() * sizeof(uint32_t); + ASSERT_GT(digest_bytes, 0U); + const size_t charged = with_digest.index.memory_usage(); + const size_t baseline = without_digest.index.memory_usage(); + ASSERT_GE(charged, baseline); + EXPECT_GE(charged - baseline, digest_bytes) + << "charge " << charged << " over baseline " << baseline << " omits " << digest_bytes + << " bytes of resident digest"; +} + // An index no gram query can reach must not carry the digest at all. It would be pure loss: // bytes in the metadata and cycles on every term, for a structure only the gram cost gate // consults. Segments of such indexes must stay byte-identical to what an earlier build diff --git a/gensrc/proto/snii.proto b/gensrc/proto/snii.proto index 70b4506441ac12..b81197a7807edd 100644 --- a/gensrc/proto/snii.proto +++ b/gensrc/proto/snii.proto @@ -66,6 +66,14 @@ message SniiLogicalIndexMetadataPB { // change the bytes of every pure-text directory. optional uint32 kind = 6; repeated SniiNamedBlobPB files = 7; // blob kinds only + // INVERTED only: this index's dictionary holds at least one entry whose posting list the + // writer dropped (stop-gram), so its body ends early and carries no locator. Writers MUST + // NOT set this field when it is false, for the same reason as `kind`: proto2 presence + // would change the bytes of every directory that drops nothing. The directory-wide + // required feature stays the fence for readers that cannot parse such an entry; this + // field is what says WHICH index needs it, which a rewrite has to know to re-declare it + // for exactly the indexes it carries over. + optional bool dropped_postings = 8; } message SniiBlobRefPB { diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy index 423abe2017496a..8a95e84c3876a9 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy @@ -155,6 +155,71 @@ suite("test_gram_metadata_inherit", "p0") { FROM test_gram_metadata_inherit WHERE tag MATCH 'fox'""" order_qt_inherited_and_new """SELECT /*+ SET_VAR(enable_match_without_inverted_index=false) */ id FROM test_gram_metadata_inherit WHERE dense LIKE '%abcdefgh%' AND tag MATCH 'fox'""" + + // A second inheritance, over a segment big enough for stop-gram to have fired. The phases + // above run on three rows, and a segment under kHighDfDigestDivisor rows never drops a + // posting, so they cannot tell whether a dictionary holding locator-less entries survives + // BUILD INDEX. This one holds 4,000 rows in a single segment, where the commonest grams + // are over the threshold and their posting lists are gone from the file. The index has to + // keep answering the same rows as a plain scan after the rewrite -- an inherited entry that + // lost its dropped-posting declaration reads as absent rather than as matching everything, + // which turns a common literal into zero rows. + sql "DROP TABLE IF EXISTS test_gram_inherit_stopped" + sql """CREATE TABLE test_gram_inherit_stopped ( + id INT, + msg VARCHAR(128), + tag VARCHAR(32), + INDEX idx_msg (msg) USING INVERTED PROPERTIES ("analyzer"="gram_inherit_dense") + ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true", + "inverted_index_storage_format"="SNII")""" + def stoppedRows = [] + for (int i = 0; i < 4000; i++) { + // Every row carries "shared_prefix_common_text", whose grams land far above the + // threshold and lose their postings; one row in a thousand carries a rare marker whose + // grams stay under it and keep theirs. + def marker = (i % 1000 == 0) ? "rare_marker_${i}" : "filler_${i}" + stoppedRows.add("(${i}, 'shared_prefix_common_text ${marker}', 'tag_${i % 3}')") + } + // One statement, so all 4,000 rows land in one segment and the floor is crossed. + sql "INSERT INTO test_gram_inherit_stopped VALUES ${stoppedRows.join(',')}" + sql "sync" + + // The inverted-index result cache is keyed by the raw query bytes, so the "after" phase + // could otherwise be served the "before" phase's bitmaps and never reach the rewritten + // container at all -- which is exactly what this phase exists to read. + sql "SET enable_inverted_index_query_cache=false" + + def checkStopped = { String phaseName -> + [false, true].each { useIndex -> + sql "SET enable_inverted_index_query=${useIndex}" + // A literal whose grams were dropped. The index cannot narrow it and must say so. + "order_qt_stopped_${phaseName}_${useIndex}_common"("""SELECT COUNT(*) + FROM test_gram_inherit_stopped WHERE msg LIKE '%shared_prefix_common%'""") + // A literal whose grams were kept, so the index really does filter here. + "order_qt_stopped_${phaseName}_${useIndex}_rare"("""SELECT id + FROM test_gram_inherit_stopped WHERE msg LIKE '%rare_marker_%' ORDER BY id""") + "order_qt_stopped_${phaseName}_${useIndex}_regexp"("""SELECT COUNT(*) + FROM test_gram_inherit_stopped WHERE msg REGEXP 'shared_prefix_[a-z]+'""") + } + sql "SET enable_inverted_index_query=true" + } + + checkStopped("before") + sql "SET enable_add_index_for_new_data=true" + sql """CREATE INDEX idx_stopped_tag ON test_gram_inherit_stopped(tag) USING INVERTED + PROPERTIES ("parser"="english")""" + sql "BUILD INDEX idx_stopped_tag ON test_gram_inherit_stopped" + wait_for_last_build_index_finish("test_gram_inherit_stopped", 180_000) + def stoppedJobs = sql_return_maparray("""SHOW BUILD INDEX + WHERE TableName='test_gram_inherit_stopped' ORDER BY JobId DESC LIMIT 1""") + if (stoppedJobs.isEmpty() || stoppedJobs[0].State != "FINISHED") { + throw new IllegalStateException("stopped-posting BUILD INDEX did not finish: ${stoppedJobs}") + } + sql "sync" + checkStopped("after") + sql "DROP TABLE IF EXISTS test_gram_inherit_stopped" + // The policies live cluster-wide and the cluster is shared, so a suite that leaves // its own behind eats into the instance-wide policy limit for everyone else. sql "DROP TABLE IF EXISTS test_gram_metadata_inherit" From 3c06bd220dd9581b7f3f3da04cfb129d7cef1b02 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 11 Sep 2026 22:40:08 +0800 Subject: [PATCH 17/36] [test](regression) give the inheritance suite's sparse column a needle it can actually index The suite asserted that both the dense and the sparse index still prune after inheritance, and the sparse half of that assertion has never held: this suite only runs on a non-cloud cluster (cloud has no named BUILD INDEX), and there it failed on every run. The cause is not the index. A sparse scheme cuts on content, so the grams of an occurrence depend on the bytes around it, and the writer solves its own density from each segment's data. On three tiny rows the solved rate is sparse enough that a bare eight-byte value promises no gram an eight-byte literal can ask for, so the compiler answers ALL -- conservative and correct. Measured on a live cluster with the default adaptive density: with the value as 'abcdefgh' the sparse index reports candidates 0, filtered 0, gave-up 0, meaning it never runs; with context around the needle it reports candidates 1, filtered 2. Turning adaptive density off makes the bare value prune again, which pins the solver as the variable. Giving the sparse column context keeps the assertion meaningful instead of weakening it. The dense column is left alone: its grams are fixed-length and need no context. Row 1 stays a dense3 false positive that is not a sparse candidate, so the results, and the golden file, are unchanged. --- .../gram/test_gram_metadata_inherit.groovy | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy index 8a95e84c3876a9..19dd353b622b07 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy @@ -68,8 +68,16 @@ suite("test_gram_metadata_inherit", "p0") { "inverted_index_storage_format"="SNII")""" // Row 1 is a dense3 false positive, but not a sparse4 candidate. Both indexes must // preserve row 0 and the scalar LIKE/REGEXP recheck must reject row 1. + // The sparse column carries context around the needle, and that is load-bearing rather than + // decoration. A sparse scheme cuts on content, so the grams of an occurrence depend on the + // bytes around it, and the writer solves its own density from the segment's data: on three + // tiny rows the solved rate is sparse enough that a bare eight-byte value promises no gram + // an eight-byte literal could ask for, and the compiler correctly answers ALL. Measured on + // a live cluster: with row 0 as 'abcdefgh' the sparse index reports candidates 0, filtered + // 0, gave-up 0 -- it never runs -- while with the context below it reports candidates 1, + // filtered 2. The dense column needs no context because its grams are fixed-length. sql """INSERT INTO test_gram_metadata_inherit VALUES - (0, 'abcdefgh', 'abcdefgh', 'red fox'), + (0, 'abcdefgh', 'prefix_abcdefgh_suffix', 'red fox'), (1, 'abc!bcd!cde!def!efg!fgh', 'abc!bcd!cde!def!efg!fgh', 'blue fox'), (2, 'unrelated', 'unrelated', 'green dog')""" sql "sync" From 1eb82590b4fada5fff77cc6672ce961184fa8d32 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Sat, 12 Sep 2026 01:42:15 +0800 Subject: [PATCH 18/36] [test](regression) assert the stopped-posting inheritance phase instead of comparing a golden The phase added with the per-index dropped-posting fix compared its queries against a golden file it has no blocks in, and it cannot get them: the suite returns early on a cloud cluster (no named BUILD INDEX there), so regenerating the golden on one emits nothing for these tags and the run fails with "Missing output block for tag 'stopped_before_false_common'". Comparing the two arms is what the phase is for, so assert that directly. The corpus is fixed, so the absolute answers are pinned as well -- 4,000 rows carry the common literal, the rare marker sits on one row in a thousand -- because parity alone would still hold for a phase that silently stopped matching anything. --- .../gram/test_gram_metadata_inherit.groovy | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy index 19dd353b622b07..81aaf6f1fd4f69 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy @@ -198,19 +198,38 @@ suite("test_gram_metadata_inherit", "p0") { // container at all -- which is exactly what this phase exists to read. sql "SET enable_inverted_index_query_cache=false" + // Answers with the index on and off, for one setting of enable_inverted_index_query. + def runStopped = { boolean useIndex -> + sql "SET enable_inverted_index_query=${useIndex}" + def out = [:] + // A literal whose grams were dropped: the index cannot narrow it and has to say so. + out.common = sql("""SELECT COUNT(*) FROM test_gram_inherit_stopped + WHERE msg LIKE '%shared_prefix_common%'""")[0][0] as long + // A literal whose grams were kept, so the index really does filter here. + out.rare = sql("""SELECT id FROM test_gram_inherit_stopped + WHERE msg LIKE '%rare_marker_%' ORDER BY id""").collect { it[0] as int } + out.regexp = sql("""SELECT COUNT(*) FROM test_gram_inherit_stopped + WHERE msg REGEXP 'shared_prefix_[a-z]+'""")[0][0] as long + return out + } + + // Asserted rather than compared against a golden file: this suite runs only on a non-cloud + // cluster, so -forceGenOut on a cloud one emits no block for these tags and the comparison + // could never be regenerated. The parity of the two arms is what the phase is for, and the + // corpus is fixed, so the absolute answers are pinned too -- without them a phase that + // silently stopped matching anything would still agree with itself. def checkStopped = { String phaseName -> - [false, true].each { useIndex -> - sql "SET enable_inverted_index_query=${useIndex}" - // A literal whose grams were dropped. The index cannot narrow it and must say so. - "order_qt_stopped_${phaseName}_${useIndex}_common"("""SELECT COUNT(*) - FROM test_gram_inherit_stopped WHERE msg LIKE '%shared_prefix_common%'""") - // A literal whose grams were kept, so the index really does filter here. - "order_qt_stopped_${phaseName}_${useIndex}_rare"("""SELECT id - FROM test_gram_inherit_stopped WHERE msg LIKE '%rare_marker_%' ORDER BY id""") - "order_qt_stopped_${phaseName}_${useIndex}_regexp"("""SELECT COUNT(*) - FROM test_gram_inherit_stopped WHERE msg REGEXP 'shared_prefix_[a-z]+'""") - } + def scanned = runStopped(false) + def indexed = runStopped(true) sql "SET enable_inverted_index_query=true" + assertEquals(scanned, indexed, + "[${phaseName}] the gram index changed the answer: ${scanned} vs ${indexed}") + assertEquals(4000L, scanned.common, + "[${phaseName}] every row carries the common literal") + assertEquals([0, 1000, 2000, 3000], scanned.rare, + "[${phaseName}] the rare marker sits on one row in a thousand") + assertEquals(4000L, scanned.regexp, + "[${phaseName}] every row matches the common regexp") } checkStopped("before") From da8063697fe97fb389f0c2e6719ac27a59ec54e6 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Sat, 12 Sep 2026 12:00:24 +0800 Subject: [PATCH 19/36] [test](regression) wait for a settled index size before comparing dropped against kept The stop-gram suite proves that postings really were dropped by comparing the index of a single 4,000-row segment against the same rows split into five segments below the drop floor. It read each size from information_schema as soon as the figure turned non-zero, but that figure is aggregated across tablets as their reports arrive, so an early sample can be a fraction of the real size. In cloud_p0 that read the dropped index as 14,426 bytes against the kept one's 7,688 -- the comparison inverted -- while the same commit's non-cloud P0 run passed the same suite with the same data. Wait for two consecutive equal non-zero readings instead. --- .../gram/test_gram_stop_gram.groovy | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy index bca81ae6b9b10f..e0eef13053873b 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy @@ -121,19 +121,24 @@ suite("test_gram_stop_gram", "p0") { } // information_schema reports a fresh table's sizes with a delay, and 0 there means "not - // yet" rather than "empty". + // yet" rather than "empty". A non-zero reading is not enough either: the figure is + // aggregated across tablets as their reports arrive, so an early sample can be a fraction + // of the real size -- which once made the comparison below read the dropped index as the + // larger of the two. Wait for the same value twice in a row before believing it. def indexBytes = { String table -> def deadline = System.currentTimeMillis() + 180_000 + long previous = -1L while (System.currentTimeMillis() < deadline) { def r = sql """SELECT INDEX_LENGTH FROM information_schema.tables WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '${table}'""" def bytes = r.isEmpty() ? 0L : (r[0][0] as long) - if (bytes > 0) { + if (bytes > 0 && bytes == previous) { return bytes } - sleep(3000) + previous = bytes + sleep(5000) } - throw new IllegalStateException("index size of ${table} never reported") + throw new IllegalStateException("index size of ${table} never settled") } def oneSegment = "test_gram_stop_gram_one" From 55c5f7f1f258ccf1bd89c310d6bc5e067b1f8885 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Sun, 13 Sep 2026 23:25:16 +0800 Subject: [PATCH 20/36] [fix](be) index declaration order no longer decides what LIKE, REGEXP and phrase queries can use Two places picked a column's index by the order the indexes were declared in rather than by what the query needs. A LIKE / REGEXP gram query carries no analyzer, so the iterator handed it the lowest-id FULLTEXT reader. With an ordinary tokenized index declared before the gram index -- which is what adding a gram index to a column that already has a MATCH index produces -- that reader answered "not a gram-family index", dispatch_query took the skip as success, and the predicate ran as a scan. Measured on a live cluster over 20,000 rows: gram index declared first, RowsGramIndexFiltered 19980; english index declared first, 0. Readers now report is_gram_family(), answered from the index properties and the in-memory policy without opening the index, and a gram query takes the lowest-id FULLTEXT reader that is gram-family, keeping the previous choice when there is none. Retrying readers on SKIPPED was rejected: that answer comes from the segment's persisted scheme, so a retry would open every ordinary index on every segment before reaching the gram one. MATCH_PHRASE / MATCH_PHRASE_PREFIX / MATCH_PHRASE_EDGE judged phrase support on the first FULLTEXT reader instead of the one the analyzer selects. A docs-only index in front -- a gram index is docs-only by default, and so is any index declared with support_phrase=false -- failed a valid USING ANALYZER query with INDEX_INVALID_PARAMETERS, which is not downgraded to a scan. The check now judges the reader read_from_index will run the query on, selected from the same column type, query type and analyzer key, and only when that reader is a FULLTEXT one. --- be/src/exprs/function/match.cpp | 19 +- .../inverted/inverted_index_iterator.cpp | 22 ++ .../index/inverted/inverted_index_reader.h | 5 + be/src/storage/index/snii/snii_index_reader.h | 6 + .../exprs/function/function_match_test.cpp | 105 +++++++++ .../segment/inverted_index_iterator_test.cpp | 63 +++++ .../gram/test_gram_index_order.groovy | 222 ++++++++++++++++++ 7 files changed, 441 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/inverted_index_p0/gram/test_gram_index_order.groovy diff --git a/be/src/exprs/function/match.cpp b/be/src/exprs/function/match.cpp index c34e6a330ed56b..4d747df0e56a55 100644 --- a/be/src/exprs/function/match.cpp +++ b/be/src/exprs/function/match.cpp @@ -62,7 +62,24 @@ Status FunctionMatchBase::evaluate_inverted_index( } const std::string& function_name = get_name(); - // support_phrase is checked once the analyzer has selected the reader that runs the query. + if (function_name == MATCH_PHRASE_FUNCTION || function_name == MATCH_PHRASE_PREFIX_FUNCTION || + function_name == MATCH_PHRASE_EDGE_FUNCTION) { + // Judge phrase support on the index the query will run on: read_from_index selects it + // from the same column type, query type and analyzer key. The first FULLTEXT reader need + // not be that index -- a docs-only one (a gram index is docs-only by default) can be + // declared ahead of the positional index USING ANALYZER names. + if (auto* inverted_iter = dynamic_cast(iter)) { + auto reader = inverted_iter->select_best_reader( + data_type_with_name.second, get_query_type_from_fn_name(), + analyzer_ctx != nullptr ? analyzer_ctx->analyzer_key : std::string()); + if (reader.has_value() && + segment_v2::IndexReaderHelper::is_fulltext_index(reader.value()) && + !segment_v2::IndexReaderHelper::is_support_phrase(reader.value())) { + return Status::Error( + "phrase queries require setting support_phrase = true"); + } + } + } Field param_value; arguments[0].column->get(0, param_value); if (param_value.is_null()) { diff --git a/be/src/storage/index/inverted/inverted_index_iterator.cpp b/be/src/storage/index/inverted/inverted_index_iterator.cpp index 85b864b4c56e7e..27a9edffa264c7 100644 --- a/be/src/storage/index/inverted/inverted_index_iterator.cpp +++ b/be/src/storage/index/inverted/inverted_index_iterator.cpp @@ -176,6 +176,28 @@ Result InvertedIndexIterator::select_best_reader( } field_type = get_inverted_index_leaf_field_type(column_type); } + // A gram query names no analyzer, so the generic order would hand it the lowest-id FULLTEXT + // reader even when that is an ordinary tokenized index, which can only decline it. Prefer the + // lowest-id reader whose analyzer cuts grams; without one the usual choice stands, and that + // reader declines the query as before. + if (is_gram_query(query_type) && normalized_key.empty() && _selection_candidates.size() > 1) { + std::optional gram_reader; + for (size_t i = 0; i < _selection_candidates.size(); ++i) { + const auto& candidate = _selection_candidates[i]; + if (candidate.reader_type != InvertedIndexReaderType::FULLTEXT || + (gram_reader.has_value() && + _selection_candidates[*gram_reader].index_id < candidate.index_id)) { + continue; + } + DORIS_CHECK(i < _readers.size()); + if (_readers[i]->is_gram_family()) { + gram_reader = i; + } + } + if (gram_reader.has_value()) { + return _readers[*gram_reader]; + } + } auto selection = select_best_inverted_index_candidate(_selection_candidates, _key_to_entries, field_type, query_type, normalized_key, legacy_analyzer_key); diff --git a/be/src/storage/index/inverted/inverted_index_reader.h b/be/src/storage/index/inverted/inverted_index_reader.h index ab21a2f93e7e31..a8bd8560f20e88 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.h +++ b/be/src/storage/index/inverted/inverted_index_reader.h @@ -272,6 +272,11 @@ class InvertedIndexReader : public IndexReader { virtual InvertedIndexReaderType type() = 0; + // Whether the index names an analyzer that cuts sparse or dense grams under the current + // policy -- the index LIKE / REGEXP gram queries are meant for. Decided from the index + // properties and in-memory policies only, never by opening the index. + virtual bool is_gram_family() const { return false; } + [[nodiscard]] uint64_t get_index_id() const override { return _index_meta.index_id(); } [[nodiscard]] MOCK_FUNCTION const std::map& get_index_properties() diff --git a/be/src/storage/index/snii/snii_index_reader.h b/be/src/storage/index/snii/snii_index_reader.h index f187796f12358f..e3eabac7824efe 100644 --- a/be/src/storage/index/snii/snii_index_reader.h +++ b/be/src/storage/index/snii/snii_index_reader.h @@ -96,6 +96,12 @@ class SniiIndexReader final : public InvertedIndexReader { InvertedIndexQueryCacheHandle* cache_handle, lucene::store::Directory* dir = nullptr) override; InvertedIndexReaderType type() override { return _reader_type; } + // A policy that cannot be resolved names no index a gram query could use; an analyzed query + // reports that failure where it matters. + bool is_gram_family() const override { + std::optional scheme; + return _current_gram_scheme(nullptr, &scheme).ok() && scheme.has_value(); + } #ifdef BE_TEST void set_single_flight_follower_joined_observer_for_test( diff --git a/be/test/exprs/function/function_match_test.cpp b/be/test/exprs/function/function_match_test.cpp index 5ec095f63cf5e7..0f06b29e2a31df 100644 --- a/be/test/exprs/function/function_match_test.cpp +++ b/be/test/exprs/function/function_match_test.cpp @@ -27,10 +27,16 @@ #include "core/block/block.h" #include "core/column/column_string.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_string.h" #include "exprs/function/match.h" #include "runtime/runtime_state.h" +#include "storage/index/index_query_context.h" #include "storage/index/inverted/analyzer/analyzer.h" #include "storage/index/inverted/analyzer/custom_analyzer.h" +#include "storage/index/inverted/inverted_index_iterator.h" +#include "storage/index/inverted/inverted_index_reader.h" +#include "storage/olap_common.h" +#include "storage/tablet/tablet_schema.h" namespace doris { @@ -876,4 +882,103 @@ TEST(FunctionMatchTest, function_registration) { EXPECT_TRUE(true); } +namespace { + +// A FULLTEXT reader that answers every query with an empty result and counts the queries it +// served, so a test can put indexes with different capabilities side by side on one column. +class CountingFulltextReader final : public segment_v2::InvertedIndexReader { +public: + static std::shared_ptr create(int64_t index_id, + const std::string& analyzer, + bool support_phrase) { + TabletIndexPB pb; + pb.set_index_id(index_id); + pb.set_index_name("idx_" + analyzer); + pb.set_index_type(IndexType::INVERTED); + pb.add_col_unique_id(0); + (*pb.mutable_properties())["analyzer"] = analyzer; + (*pb.mutable_properties())["support_phrase"] = support_phrase ? "true" : "false"; + TabletIndex index; + index.init_from_pb(pb); + return std::shared_ptr(new CountingFulltextReader(index)); + } + + segment_v2::InvertedIndexReaderType type() override { + return segment_v2::InvertedIndexReaderType::FULLTEXT; + } + Status query(const segment_v2::IndexQueryContextPtr& /*context*/, + const std::string& /*column_name*/, const Field& /*query_value*/, + segment_v2::InvertedIndexQueryType /*query_type*/, + std::shared_ptr& /*bit_map*/, + const InvertedIndexAnalyzerCtx* /*analyzer_ctx*/) override { + ++queries; + return Status::OK(); + } + Status try_query(const segment_v2::IndexQueryContextPtr& /*context*/, + const std::string& /*column_name*/, const Field& /*query_value*/, + segment_v2::InvertedIndexQueryType /*query_type*/, size_t* count) override { + *count = 0; + return Status::OK(); + } + Status new_iterator(std::unique_ptr* /*iterator*/) override { + return Status::OK(); + } + + int queries = 0; + +private: + explicit CountingFulltextReader(const TabletIndex& index) + : segment_v2::InvertedIndexReader(&index, nullptr) { + // No index file stands behind this reader, so there is no null bitmap to read either. + set_has_null(false); + } +}; + +} // namespace + +// A column can carry a docs-only index -- a gram index is docs-only by default -- ahead of a +// positional one. MATCH_PHRASE ... USING ANALYZER names the positional index and the query runs +// on it, so phrase support has to be judged on that index. Judging it on the first FULLTEXT +// reader rejected a valid query purely because of the order the indexes were created in. +TEST(FunctionMatchTest, phrase_support_is_checked_on_the_index_the_analyzer_selects) { + auto docs_only = CountingFulltextReader::create(1, "docs_only_analyzer", false); + auto positional = CountingFulltextReader::create(2, "positional_analyzer", true); + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, docs_only); + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, positional); + OlapReaderStatistics stats; + auto context = std::make_shared(); + context->stats = &stats; + iterator.set_context(context); + + const auto string_type = std::make_shared(); + const ColumnsWithTypeAndName arguments { + {string_type->create_column_const( + 1, Field::create_field(std::string("request ok"))), + string_type, "query"}}; + const std::vector columns {{"msg", string_type}}; + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.analyzer_key = "positional_analyzer"; + + FunctionMatchPhrase match_phrase; + FunctionMatchPhrasePrefix match_phrase_prefix; + for (const FunctionMatchBase* function : + std::vector {&match_phrase, &match_phrase_prefix}) { + segment_v2::InvertedIndexResultBitmap bitmap; + const auto status = function->evaluate_inverted_index(arguments, columns, {&iterator}, 4, + &analyzer_ctx, bitmap); + EXPECT_TRUE(status.ok()) << function->get_name() << ": " << status; + } + EXPECT_EQ(positional->queries, 2); + EXPECT_EQ(docs_only->queries, 0); + + // Naming the docs-only index still gets the phrase query refused. + analyzer_ctx.analyzer_key = "docs_only_analyzer"; + segment_v2::InvertedIndexResultBitmap bitmap; + const auto refused = match_phrase.evaluate_inverted_index(arguments, columns, {&iterator}, 4, + &analyzer_ctx, bitmap); + EXPECT_TRUE(refused.is()) << refused; + EXPECT_EQ(docs_only->queries, 0); +} + } // namespace doris diff --git a/be/test/storage/segment/inverted_index_iterator_test.cpp b/be/test/storage/segment/inverted_index_iterator_test.cpp index 09de1b596c4785..279679668d95c4 100644 --- a/be/test/storage/segment/inverted_index_iterator_test.cpp +++ b/be/test/storage/segment/inverted_index_iterator_test.cpp @@ -84,6 +84,9 @@ class MockInvertedIndexReader : public InvertedIndexReader { Status new_iterator(std::unique_ptr* iterator) override { return Status::OK(); } + bool is_gram_family() const override { return _gram_family; } + void set_gram_family(bool gram_family) { _gram_family = gram_family; } + private: MockInvertedIndexReader(std::shared_ptr index, const std::map& properties) @@ -94,6 +97,7 @@ class MockInvertedIndexReader : public InvertedIndexReader { std::shared_ptr _mock_index; // Keep index alive std::map _properties; InvertedIndexReaderType _type = InvertedIndexReaderType::FULLTEXT; + bool _gram_family = false; }; class InvertedIndexIteratorTest : public testing::Test { @@ -957,4 +961,63 @@ TEST_F(InvertedIndexIteratorTest, PhraseQueriesStillRunOnAnUntokenizedIndex) { } } +// A gram query carries no analyzer, so it cannot name the index it is meant for. An ordinary +// tokenized index declared before the gram index holds the lower id; handing it the query gets +// back "not a gram-family index" and silently drops the acceleration the gram index exists for. +TEST_F(InvertedIndexIteratorTest, SelectBestReader_GramQueryPrefersGramFamilyOverOlderIndex) { + auto english = create_mock_reader("english", InvertedIndexReaderType::FULLTEXT, 10); + auto gram = create_mock_reader("my_gram_analyzer", InvertedIndexReaderType::FULLTEXT, 20); + gram->set_gram_family(true); + InvertedIndexIterator iter; + iter.add_reader(InvertedIndexReaderType::FULLTEXT, english); + iter.add_reader(InvertedIndexReaderType::FULLTEXT, gram); + + auto col_type = std::make_shared(); + for (auto query_type : + {InvertedIndexQueryType::LIKE_GRAM_QUERY, InvertedIndexQueryType::REGEXP_GRAM_QUERY}) { + SCOPED_TRACE(query_type_to_string(query_type)); + auto result = iter.select_best_reader(col_type, query_type, ""); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), gram); + } + // Analyzed queries keep choosing by id: the preference only serves queries that cannot name + // their index. + auto match = iter.select_best_reader(col_type, InvertedIndexQueryType::MATCH_ANY_QUERY, ""); + ASSERT_TRUE(match.has_value()); + EXPECT_EQ(match.value(), english); +} + +// Several gram-family indexes still resolve by index id, and a column without any keeps the +// ordinary choice: that reader declines the gram query and the predicate runs without the index, +// exactly as before. +TEST_F(InvertedIndexIteratorTest, SelectBestReader_GramQueryKeepsIdOrderAndFallsBack) { + auto col_type = std::make_shared(); + { + auto english = create_mock_reader("english", InvertedIndexReaderType::FULLTEXT, 5); + auto newer_gram = create_mock_reader("gram_b", InvertedIndexReaderType::FULLTEXT, 30); + auto older_gram = create_mock_reader("gram_a", InvertedIndexReaderType::FULLTEXT, 20); + newer_gram->set_gram_family(true); + older_gram->set_gram_family(true); + InvertedIndexIterator iter; + iter.add_reader(InvertedIndexReaderType::FULLTEXT, english); + iter.add_reader(InvertedIndexReaderType::FULLTEXT, newer_gram); + iter.add_reader(InvertedIndexReaderType::FULLTEXT, older_gram); + auto result = + iter.select_best_reader(col_type, InvertedIndexQueryType::LIKE_GRAM_QUERY, ""); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), older_gram); + } + { + auto standard = create_mock_reader("standard", InvertedIndexReaderType::FULLTEXT, 7); + auto english = create_mock_reader("english", InvertedIndexReaderType::FULLTEXT, 5); + InvertedIndexIterator iter; + iter.add_reader(InvertedIndexReaderType::FULLTEXT, standard); + iter.add_reader(InvertedIndexReaderType::FULLTEXT, english); + auto result = + iter.select_best_reader(col_type, InvertedIndexQueryType::REGEXP_GRAM_QUERY, ""); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), english); + } +} + } // namespace doris::segment_v2 diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_index_order.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_index_order.groovy new file mode 100644 index 00000000000000..9efc7bb489a93d --- /dev/null +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_index_order.groovy @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.regex.Pattern + +import org.apache.doris.regression.action.ProfileAction + +// The order in which a column's indexes were declared must not decide what a query can use. +// Two places where it did: +// 1) LIKE / REGEXP carry no analyzer, so they were handed to the lowest-id tokenized index. An +// ordinary index declared before the gram index then declined the gram query, and the +// predicate quietly ran as a scan: the same rows, none of the acceleration. +// 2) MATCH_PHRASE ... USING ANALYZER names a positional index, but phrase support was judged on +// whichever tokenized index came first. A docs-only index in that place -- and a gram index +// is docs-only by default -- made a valid query fail. +suite("test_gram_index_order", "p0") { + def sparseTok = "gram_order_sparse_tok" + def sparseAna = "gram_order_sparse" + def positionalAna = "gram_order_positional" + def likeTbl = "t_gram_order_like" + def gramPhraseTbl = "t_gram_order_phrase_gram" + def englishPhraseTbl = "t_gram_order_phrase_english" + + // An analyzer reaches BE asynchronously over the heartbeat; wait until BE can use it. + def waitAnalyzerInstalled = { String name -> + def deadline = System.currentTimeMillis() + 180_000 + Exception lastNotFound = null + while (System.currentTimeMillis() < deadline) { + try { + sql """SELECT TOKENIZE('probe', '\"analyzer\"=\"${name}\"')""" + return + } catch (Exception e) { + if (!e.message.contains("Policy not found")) { + throw e + } + lastNotFound = e + sleep(1000) + } + } + throw new IllegalStateException("analyzer ${name} was not installed on BE", lastNotFound) + } + def idsOf = { String query -> + return sql(query).collect { it[0] as Integer }.sort() + } + // A counter may be rendered as "18" or as "12.0K (12000)", and both forms must be parseable. + def parseProfileCounter = { String profileString, String name -> + def exact = Pattern.compile(Pattern.quote(name) + ":\\s*[^\\(\\n]*\\((\\d+)\\)").matcher(profileString) + if (exact.find()) { + return Long.parseLong(exact.group(1)) + } + def plain = Pattern.compile(Pattern.quote(name) + ":\\s*(\\d+)").matcher(profileString) + assertTrue(plain.find(), "${name} is not parseable from profile") + return Long.parseLong(plain.group(1)) + } + + sql "SET enable_sql_cache=false" + // Both passes of every parity check must really scan; see test_gram_regexp_like for why the + // condition cache would otherwise let them share one filter result. + sql "SET enable_condition_cache=false" + // With function push-down on, LIKE becomes a storage-layer predicate and never reaches the + // index, so the profile assertion below would see nothing pruned. + sql "SET enable_function_pushdown=false" + + [likeTbl, gramPhraseTbl, englishPhraseTbl].each { sql "DROP TABLE IF EXISTS ${it}" } + try_sql "DROP INVERTED INDEX ANALYZER IF EXISTS ${sparseAna}" + try_sql "DROP INVERTED INDEX ANALYZER IF EXISTS ${positionalAna}" + try_sql "DROP INVERTED INDEX TOKENIZER IF EXISTS ${sparseTok}" + sql """ + CREATE INVERTED INDEX TOKENIZER IF NOT EXISTS ${sparseTok} + PROPERTIES ( + "type" = "ngram", + "mode" = "sparse", + "min_gram" = "3", + "max_gram" = "16", + "density" = "0.25" + ) + """ + sql """ + CREATE INVERTED INDEX ANALYZER IF NOT EXISTS ${sparseAna} + PROPERTIES ("tokenizer" = "${sparseTok}") + """ + sql """ + CREATE INVERTED INDEX ANALYZER IF NOT EXISTS ${positionalAna} + PROPERTIES ("tokenizer" = "standard") + """ + waitAnalyzerInstalled(sparseAna) + waitAnalyzerInstalled(positionalAna) + + // 1) An ordinary english index declared first, so it holds the lower index id. + sql """ + CREATE TABLE ${likeTbl} ( + id INT, + msg VARCHAR(512), + INDEX idx_msg_en (msg) USING INVERTED PROPERTIES ("parser" = "english"), + INDEX idx_msg_gram (msg) USING INVERTED PROPERTIES ("analyzer" = "${sparseAna}") + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "inverted_index_storage_format" = "SNII" + ) + """ + sql """INSERT INTO ${likeTbl} VALUES + (1, 'rpc error: code = Unavailable desc = error reading from server'), + (2, 'user_id="eacb47f6-967d-11f0-b88d-8eb93cba8bdb" user_currency="USD"'), + (3, 'Convert conversion successful'), + (4, 'mobile POST 10.68.3.18:8080 error'), + (5, NULL), + (6, ''), + (7, 'ab'), + (8, 'GET /images/x.gif HTTP/1.0'), + (9, 'CODE = UNAVAILABLE'), + (10, 'context deadline exceeded'), + (11, 'failed to charge card: rpc error'), + (12, 'timeout after error error error')""" + sql "sync" + sql """INSERT INTO ${likeTbl} VALUES + (13, 'rpc error: code = Internal desc = boom'), + (14, 'mobile client upload'), + (15, 'abc'), + (16, 'Sending Quote: 12.5'), + (17, 'progress 100% done'), + (18, ' '), + (19, 'MiXeD CaSe UnAvAiLaBlE')""" + sql "sync" + + def profileAction = new ProfileAction(context) + [ + ["like", "msg LIKE '%Sending Quote%'", [16]], + ["regexp", "msg REGEXP 'context deadline exceeded'", [10]], + ].each { entry -> + def label = entry[0] + def query = "SELECT id FROM ${likeTbl} WHERE ${entry[1]}".toString() + sql "SET enable_inverted_index_query=false" + def withoutIndex = idsOf(query) + sql "SET enable_inverted_index_query=true" + def withIndex = idsOf(query) + assertEquals(entry[2], withoutIndex, "[${label}] scalar answer") + assertEquals(withoutIndex, withIndex, "[${label}] the index changed the answer") + + // The same rows are not enough: the gram index declared second must actually prune. + def tag = "gram_order_${label}".toString() + sql "set enable_profile=true" + sql "set profile_level=2" + sql "/* ${tag} */ ${query}" + def profileString = profileAction.getProfileBySql(tag, + ["RowsGramIndexFiltered", "GramIndexCandidateRows"]) + sql "set enable_profile=false" + def filtered = parseProfileCounter(profileString, "RowsGramIndexFiltered") + log.info("[${label}] RowsGramIndexFiltered=${filtered}".toString()) + assertTrue(filtered > 0, + "[${label}] with an ordinary index declared first the gram index must still prune, " + + "RowsGramIndexFiltered=${filtered}") + } + + // 2) A docs-only index declared before a positional one: the gram index, docs-only by + // default, and for contrast an ordinary english index made docs-only explicitly. + [ + [gramPhraseTbl, + """INDEX idx_msg_docs (msg) USING INVERTED PROPERTIES ("analyzer" = "${sparseAna}")"""], + [englishPhraseTbl, + """INDEX idx_msg_docs (msg) USING INVERTED PROPERTIES ("parser" = "english", "support_phrase" = "false")"""], + ].each { entry -> + def tbl = entry[0] + sql """ + CREATE TABLE ${tbl} ( + id INT, + msg VARCHAR(512), + ${entry[1]}, + INDEX idx_msg_positional (msg) USING INVERTED PROPERTIES ( + "analyzer" = "${positionalAna}", "support_phrase" = "true") + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "inverted_index_storage_format" = "SNII" + ) + """ + sql """INSERT INTO ${tbl} VALUES + (1, 'request ok here'), (2, 'ok request'), (3, 'nothing'), (4, 'a request ok'), + (5, NULL)""" + sql "sync" + [ + "msg MATCH_PHRASE 'request ok' USING ANALYZER ${positionalAna}", + "msg MATCH_PHRASE_PREFIX 'request o' USING ANALYZER ${positionalAna}", + ].each { predicate -> + def query = "SELECT id FROM ${tbl} WHERE ${predicate}".toString() + sql "SET enable_inverted_index_query=false" + def withoutIndex = idsOf(query) + sql "SET enable_inverted_index_query=true" + def withIndex = idsOf(query) + assertEquals([1, 4], withoutIndex, "[${tbl}] scalar answer for ${predicate}") + assertEquals(withoutIndex, withIndex, "[${tbl}] index answer for ${predicate}") + } + } + + sql "SET enable_inverted_index_query=true" + // The policies live cluster-wide and the cluster is shared, so a suite that leaves + // its own behind eats into the instance-wide policy limit for everyone else. + [likeTbl, gramPhraseTbl, englishPhraseTbl].each { sql "DROP TABLE IF EXISTS ${it}" } + sql "DROP INVERTED INDEX ANALYZER IF EXISTS ${sparseAna}" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS ${positionalAna}" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS ${sparseTok}" +} From a88790d57bc96ced2efeaad97fc6474b660b548c Mon Sep 17 00:00:00 2001 From: airborne12 Date: Sun, 13 Sep 2026 23:25:16 +0800 Subject: [PATCH 21/36] [fix](be) gram scheme fence also covers a gram segment read under a scheme-less analyzer The fence skipped an analyzed query only when the current analyzer had a gram scheme. A gram segment recovered after its analyzer name was recreated for a tokenizer that predates gram mode has it the other way round: the segment carries a scheme and the analyzer does not, so the query's legacy terms were looked up in a gram dictionary. On a live cluster MATCH_ANY 'abcd' returned [] with the index on and [1,2,4,5] with it off, and MATCH_ALL and MATCH_REGEXP did the same. The result cache made it stick: with no current scheme the cache is enabled, so the first wrong bitmap was inserted and the next identical query was served from it. Compare the two optionals whichever side is empty. The fence returns before a bitmap is computed, so nothing mismatched is ever inserted, and while the analyzer still had a scheme the cache was not consulted at all. --- .../storage/index/snii/snii_index_reader.cpp | 14 +++-- .../index/snii/snii_gram_cache_test.cpp | 40 +++++++++++++ .../gram/test_gram_policy_recovery.groovy | 58 +++++++++++++++++++ 3 files changed, 106 insertions(+), 6 deletions(-) diff --git a/be/src/storage/index/snii/snii_index_reader.cpp b/be/src/storage/index/snii/snii_index_reader.cpp index bf5edb08d9a8fb..05a0f0e822beea 100644 --- a/be/src/storage/index/snii/snii_index_reader.cpp +++ b/be/src/storage/index/snii/snii_index_reader.cpp @@ -771,12 +771,14 @@ Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::s const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, &logical_reader)); - // Compare the two optionals, not just two schemes: a segment written by a legacy ngram - // tokenizer carries no scheme at all, and its dictionary holds that tokenizer's terms. Once - // the current analyzer cuts grams, looking those grams up in that dictionary answers a - // different question, so an absent persisted scheme is a mismatch like any other. - if (analyzed_query && current_gram_scheme.has_value() && - current_gram_scheme != logical_reader->gram_scheme()) { + // Compare the two optionals, not just two schemes, and whichever side is empty. A segment + // written by a legacy ngram tokenizer carries no scheme, and its dictionary holds that + // tokenizer's terms, so grams cut by the current analyzer mean nothing there. The reverse + // holds too: a gram segment recovered after its policy was recreated without a mode keeps its + // gram dictionary while the analyzer now cuts legacy terms -- and with no current scheme the + // result cache is enabled. Returning here, before anything is computed, keeps such an answer + // out of the index and out of that cache alike. + if (analyzed_query && current_gram_scheme != logical_reader->gram_scheme()) { return Status::Error( "gram index segment was cut with a different scheme than the current " "analyzer; the predicate is evaluated without the index"); diff --git a/be/test/storage/index/snii/snii_gram_cache_test.cpp b/be/test/storage/index/snii/snii_gram_cache_test.cpp index f20b722ae1712e..cad00716b83b9b 100644 --- a/be/test/storage/index/snii/snii_gram_cache_test.cpp +++ b/be/test/storage/index/snii/snii_gram_cache_test.cpp @@ -480,6 +480,46 @@ TEST_F(SniiGramCacheTest, UnprunablePatternsReleaseFollowersWithoutCachingResult } } +// A dense3 segment recovered after its analyzer name was recreated for a tokenizer that predates +// gram mode. The current analyzer then carries no scheme while the segment still does, and the +// segment's dictionary holds trigrams, not the bigrams the query is now cut into. The index must +// decline the analyzed query instead of answering it from that dictionary, and because a declined +// attempt computes nothing, repeating the query must decline again rather than be served from the +// result cache. +TEST_F(SniiGramCacheTest, AnalyzedQueriesDeclineWhenTheAnalyzerNoLongerCutsGrams) { + TIndexPolicy tokenizer; + tokenizer.id = 6753980; + tokenizer.name = "gram_cache_dense3_tokenizer"; + tokenizer.type = TIndexPolicyType::TOKENIZER; + tokenizer.properties = {{"type", "ngram"}, {"min_gram", "2"}, {"max_gram", "2"}}; + TIndexPolicy analyzer; + analyzer.id = tokenizer.id + 1; + analyzer.name = "gram_cache_dense3_analyzer"; + analyzer.type = TIndexPolicyType::ANALYZER; + analyzer.properties = {{"tokenizer", tokenizer.name}}; + // Replace the dense3 pair SetUp installed (ids 6753900 / 6753901) under the same names. + _policy_mgr.apply_policy_changes({tokenizer, analyzer}, {6753900, 6753901}); + ASSERT_FALSE( + _policy_mgr.get_analyzer_provider_by_name(analyzer.name)->gram_scheme().has_value()); + + for (const auto& [pattern, type] : std::vector> { + {"abcdef", InvertedIndexQueryType::MATCH_ANY_QUERY}, + {"abcdef", InvertedIndexQueryType::MATCH_ALL_QUERY}, + {"^ab$", InvertedIndexQueryType::MATCH_REGEXP_QUERY}}) { + SCOPED_TRACE(query_type_to_string(type)); + const std::vector requests {{_readers[0], pattern, type}}; + for (size_t attempt = 0; attempt < 2; ++attempt) { + SCOPED_TRACE(attempt); + const auto results = run_queries(requests); + const auto& result = results.front(); + EXPECT_TRUE(result.status.is()) + << result.status; + EXPECT_EQ(result.stats.inverted_index_query_cache_hit, 0); + EXPECT_EQ(result.stats.inverted_index_query_cache_insert, 0); + } + } +} + class SniiGramCacheOptionsTest : public SniiGramCacheTest, public testing::WithParamInterface> {}; diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy index 3763a38fa9f4c3..a2083eafca9b32 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy @@ -181,6 +181,64 @@ suite("test_gram_policy_recovery", "p0") { sql "DROP TABLE IF EXISTS test_gram_legacy_recovery" sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_legacy_analyzer" sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_legacy_tokenizer" + + // The same mismatch seen from the other side: a segment cut in gram mode, recovered after its + // policy was recreated for a tokenizer that predates gram mode. The current analyzer now + // carries no scheme while the segment still does, and the segment's dictionary holds dense3 + // grams, not the bigrams a MATCH query is cut into, so the index has to step aside here too. + // The bigram answers are pinned: parity alone would also hold if both passes came back empty. + sql "DROP TABLE IF EXISTS test_gram_reverse_recovery" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_reverse_analyzer" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_reverse_tokenizer" + sql """CREATE INVERTED INDEX TOKENIZER gram_reverse_tokenizer PROPERTIES ( + "type"="ngram", "mode"="dense", "min_gram"="3")""" + sql """CREATE INVERTED INDEX ANALYZER gram_reverse_analyzer + PROPERTIES ("tokenizer"="gram_reverse_tokenizer")""" + waitAnalyzerInstalledNamed("gram_reverse_analyzer", 4) + + sql """CREATE TABLE test_gram_reverse_recovery ( + id INT, + msg VARCHAR(128), + INDEX idx_msg (msg) USING INVERTED PROPERTIES ("analyzer"="gram_reverse_analyzer") + ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true", + "inverted_index_storage_format"="SNII")""" + sql """INSERT INTO test_gram_reverse_recovery VALUES + (1, 'abcd'), (2, 'xx abcd yy'), (3, 'nothing here'), (4, 'abce'), (5, 'zz bc zz'), + (6, NULL)""" + sql "sync" + + sql "DROP TABLE test_gram_reverse_recovery" + sql "DROP INVERTED INDEX ANALYZER gram_reverse_analyzer" + sql "DROP INVERTED INDEX TOKENIZER gram_reverse_tokenizer" + sql """CREATE INVERTED INDEX TOKENIZER gram_reverse_tokenizer PROPERTIES ( + "type"="ngram", "min_gram"="2", "max_gram"="2")""" + sql """CREATE INVERTED INDEX ANALYZER gram_reverse_analyzer + PROPERTIES ("tokenizer"="gram_reverse_tokenizer")""" + waitAnalyzerInstalledNamed("gram_reverse_analyzer", 5) + sql "RECOVER TABLE test_gram_reverse_recovery" + + def reverseIds = { String predicate -> + return sql("SELECT id FROM test_gram_reverse_recovery WHERE ${predicate} ORDER BY id") + .collect { it[0] as Integer } + } + // Bigrams of 'abcd' are ab, bc, cd; '^ab$' matches the term ab. + [ + ["msg MATCH_ANY 'abcd'", [1, 2, 4, 5]], + ["msg MATCH_ALL 'abcd'", [1, 2]], + ["msg MATCH_REGEXP '^ab\$'", [1, 2, 4]], + ["msg LIKE '%abcd%'", [1, 2]], + ].each { entry -> + sql "SET enable_inverted_index_query=false" + def withoutIndex = reverseIds(entry[0]) + sql "SET enable_inverted_index_query=true" + def withIndex = reverseIds(entry[0]) + assertEquals(entry[1], withoutIndex, "scalar answer for ${entry[0]}") + assertEquals(withoutIndex, withIndex, "the index changed the answer for ${entry[0]}") + } + sql "DROP TABLE IF EXISTS test_gram_reverse_recovery" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_reverse_analyzer" + sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_reverse_tokenizer" // The policies live cluster-wide and the cluster is shared, so a suite that leaves // its own behind eats into the instance-wide policy limit for everyone else. sql "DROP TABLE IF EXISTS test_gram_policy_recovery" From 801776f88e0c402dbd8da9afa24e9dda94e92bae Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 14 Sep 2026 08:21:07 +0800 Subject: [PATCH 22/36] [fix](be) SEARCH is never answered by a gram index, and analyzed EQUAL respects the scheme fence SEARCH's TERM and EXACT clauses reach an SNII reader as EQUAL_QUERY with the value cut by the current analyzer, but EQUAL was outside the gram scheme fence and its result-cache exclusion. On a sparse gram index that is wrong in ordinary use, not only after a policy is recreated: every segment records the density it solved from its own rows, so the value's grams are looked up in a dictionary cut at another density. Measured on a live cluster over 60,000 rows, search('msg:images') returned 0 rows where MATCH_ANY evaluated row by row returned 17,143. After a dense3 segment was recovered under a same-name dense4 analyzer, SEARCH returned [] with the result cache off and the pre-recovery bitmap [1,2,5,6] with it on, where the current analyzer answers [1,2]. A gram index only accelerates LIKE and REGEXP, and SEARCH has no row fallback, so fencing alone would have turned the sparse case into an error on every query. The SEARCH binding now refuses a gram-family reader, for every clause type, with an error that says why. The SNII reader also treats EQUAL on a tokenized reader as an analyzed query, so a mismatched scheme steps aside and never fills the scheme-blind result cache for any other caller. --- .../variant_inverted_index_search.cpp | 11 +++++ .../storage/index/snii/snii_index_reader.cpp | 9 ++-- .../exprs/function/function_search_test.cpp | 41 +++++++++++++++++++ .../index/snii/snii_gram_cache_test.cpp | 5 ++- .../gram/test_gram_index_order.groovy | 32 ++++++++++++++- .../gram/test_gram_policy_recovery.groovy | 11 +++++ 6 files changed, 104 insertions(+), 5 deletions(-) diff --git a/be/src/exprs/function/variant_inverted_index_search.cpp b/be/src/exprs/function/variant_inverted_index_search.cpp index 87abd534fcbd86..cb55a8546d40fa 100644 --- a/be/src/exprs/function/variant_inverted_index_search.cpp +++ b/be/src/exprs/function/variant_inverted_index_search.cpp @@ -229,6 +229,17 @@ Status FieldReaderResolver::resolve(const std::string& field_name, "selected reader is null for field '{}'", field_name); } + // A gram index only accelerates LIKE / REGEXP. SEARCH cuts its value with the current + // analyzer and has no row fallback, while a gram segment was cut by the scheme it recorded -- + // for a sparse index, a density solved from its own rows -- so a gram answer to SEARCH could + // silently miss rows. Refuse it outright instead of returning such a bitmap. + if (inverted_reader->is_gram_family()) { + return Status::Error( + "SEARCH cannot use gram index '{}' on field '{}': a gram index only accelerates " + "LIKE and REGEXP", + inverted_reader->get_index_meta().index_name(), field_name); + } + FieldReaderBinding resolved; resolved.logical_field_name = field_name; resolved.stored_field_name = stored_field_name; diff --git a/be/src/storage/index/snii/snii_index_reader.cpp b/be/src/storage/index/snii/snii_index_reader.cpp index 05a0f0e822beea..1c2f5857c3192e 100644 --- a/be/src/storage/index/snii/snii_index_reader.cpp +++ b/be/src/storage/index/snii/snii_index_reader.cpp @@ -731,13 +731,16 @@ Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::s if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { parse_phrase_slop(&plain_analysis_str, &query_info); } - // An analyzed query (MATCH_*) on a gram-family index is exact only when the query is cut - // by the very scheme that cut the segment. The segment's scheme is persisted -- a density + // An analyzed query (MATCH_*, and EQUAL on a tokenized reader, which is how SEARCH's TERM + // and EXACT clauses arrive) on a gram-family index is exact only when the query is cut by + // the very scheme that cut the segment. The segment's scheme is persisted -- a density // solved from its own rows, or the scheme of a policy that has since been recreated -- // while the analyzer here is the current one. Where they differ the scalar predicate and // the index would disagree, so such a query is skipped once the segment is open, and it // is never cached: the cache key does not tell the two schemes apart. - const bool analyzed_query = analyzes_query_terms(query_type) && + const bool analyzed_query = (analyzes_query_terms(query_type) || + (query_type == InvertedIndexQueryType::EQUAL_QUERY && + _reader_type == InvertedIndexReaderType::FULLTEXT)) && (analyzer_ctx == nullptr || analyzer_ctx->requires_analysis()); std::optional current_gram_scheme; if (analyzed_query) { diff --git a/be/test/exprs/function/function_search_test.cpp b/be/test/exprs/function/function_search_test.cpp index 31b1a701423851..5b5ba3225ec06d 100644 --- a/be/test/exprs/function/function_search_test.cpp +++ b/be/test/exprs/function/function_search_test.cpp @@ -181,6 +181,9 @@ class DummyInvertedIndexReader final : public segment_v2::InvertedIndexReader { } segment_v2::InvertedIndexReaderType type() override { return _reader_type; } + bool is_gram_family() const override { return gram_family; } + + bool gram_family = false; private: segment_v2::InvertedIndexReaderType _reader_type = segment_v2::InvertedIndexReaderType::BKD; @@ -459,6 +462,44 @@ static Status resolve_non_variant_binding_with_mismatched_analyzer(const DataTyp return resolver.resolve("content", InvertedIndexQueryType::MATCH_ANY_QUERY, &binding); } +// A gram index only accelerates LIKE / REGEXP. SEARCH cuts its value with the current analyzer +// and cannot fall back to rows, so a gram index would answer it from a dictionary cut by another +// scheme whenever a segment solved its own density -- the normal case for a sparse index -- and +// silently miss rows. Binding refuses such a reader for every clause type. +TEST_F(FunctionSearchTest, TestFieldReaderResolverRejectsGramIndex) { + const std::map index_properties { + {INVERTED_INDEX_ANALYZER_NAME_KEY, "gram_sparse_analyzer"}}; + auto index_meta = make_test_inverted_index(14, index_properties); + auto reader = std::make_shared( + &index_meta, nullptr, segment_v2::InvertedIndexReaderType::FULLTEXT); + reader->gram_family = true; + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "content", IndexFieldNameAndTypePair {"content", std::make_shared()}); + std::unordered_map iterators; + iterators["content"] = &iterator; + TSearchFieldBinding field_binding; + field_binding.field_name = "content"; + field_binding.index_properties = index_properties; + field_binding.__isset.index_properties = true; + + for (auto query_type : + {InvertedIndexQueryType::MATCH_ANY_QUERY, InvertedIndexQueryType::EQUAL_QUERY, + InvertedIndexQueryType::WILDCARD_QUERY}) { + SCOPED_TRACE(segment_v2::query_type_to_string(query_type)); + auto context = std::make_shared(); + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + FieldReaderBinding binding; + const auto status = resolver.resolve("content", query_type, &binding); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, status.code()) << status; + EXPECT_NE(status.to_string().find("gram index"), std::string::npos) << status; + } +} + TEST_F(FunctionSearchTest, TestGetName) { EXPECT_EQ("search", function_search->get_name()); } diff --git a/be/test/storage/index/snii/snii_gram_cache_test.cpp b/be/test/storage/index/snii/snii_gram_cache_test.cpp index cad00716b83b9b..a1579f5f7fcc15 100644 --- a/be/test/storage/index/snii/snii_gram_cache_test.cpp +++ b/be/test/storage/index/snii/snii_gram_cache_test.cpp @@ -502,10 +502,13 @@ TEST_F(SniiGramCacheTest, AnalyzedQueriesDeclineWhenTheAnalyzerNoLongerCutsGrams ASSERT_FALSE( _policy_mgr.get_analyzer_provider_by_name(analyzer.name)->gram_scheme().has_value()); + // EQUAL is how SEARCH's TERM and EXACT clauses reach a tokenized reader; its value is cut + // by the analyzer just like a MATCH value. for (const auto& [pattern, type] : std::vector> { {"abcdef", InvertedIndexQueryType::MATCH_ANY_QUERY}, {"abcdef", InvertedIndexQueryType::MATCH_ALL_QUERY}, - {"^ab$", InvertedIndexQueryType::MATCH_REGEXP_QUERY}}) { + {"^ab$", InvertedIndexQueryType::MATCH_REGEXP_QUERY}, + {"abcdef", InvertedIndexQueryType::EQUAL_QUERY}}) { SCOPED_TRACE(query_type_to_string(type)); const std::vector requests {{_readers[0], pattern, type}}; for (size_t attempt = 0; attempt < 2; ++attempt) { diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_index_order.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_index_order.groovy index 9efc7bb489a93d..244ae38ccf5f3b 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_index_order.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_index_order.groovy @@ -34,6 +34,7 @@ suite("test_gram_index_order", "p0") { def likeTbl = "t_gram_order_like" def gramPhraseTbl = "t_gram_order_phrase_gram" def englishPhraseTbl = "t_gram_order_phrase_english" + def gramOnlyTbl = "t_gram_order_search_gram" // An analyzer reaches BE asynchronously over the heartbeat; wait until BE can use it. def waitAnalyzerInstalled = { String name -> @@ -212,10 +213,39 @@ suite("test_gram_index_order", "p0") { } } + // 3) SEARCH is never answered by a gram index. A gram index only accelerates LIKE / REGEXP: a + // SEARCH value is cut by the current analyzer while a sparse segment records the density it + // solved for itself, and SEARCH has no row fallback, so a gram answer could silently miss + // rows. A binding that lands on a gram index is refused with a clear error; one that lands on + // an ordinary tokenized index -- the english index declared first above -- keeps working. + sql "SET enable_inverted_index_query=true" + sql "DROP TABLE IF EXISTS ${gramOnlyTbl}" + sql """ + CREATE TABLE ${gramOnlyTbl} ( + id INT, + msg VARCHAR(512), + INDEX idx_msg_gram (msg) USING INVERTED PROPERTIES ("analyzer" = "${sparseAna}") + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "inverted_index_storage_format" = "SNII" + ) + """ + sql """INSERT INTO ${gramOnlyTbl} VALUES (1, 'context deadline exceeded'), (2, 'request ok')""" + sql "sync" + test { + sql "SELECT id FROM ${gramOnlyTbl} WHERE search('msg:deadline')" + exception "gram index" + } + assertEquals([10], idsOf("SELECT id FROM ${likeTbl} WHERE search('msg:deadline')".toString())) + sql "SET enable_inverted_index_query=true" // The policies live cluster-wide and the cluster is shared, so a suite that leaves // its own behind eats into the instance-wide policy limit for everyone else. - [likeTbl, gramPhraseTbl, englishPhraseTbl].each { sql "DROP TABLE IF EXISTS ${it}" } + [likeTbl, gramPhraseTbl, englishPhraseTbl, gramOnlyTbl].each { sql "DROP TABLE IF EXISTS ${it}" } sql "DROP INVERTED INDEX ANALYZER IF EXISTS ${sparseAna}" sql "DROP INVERTED INDEX ANALYZER IF EXISTS ${positionalAna}" sql "DROP INVERTED INDEX TOKENIZER IF EXISTS ${sparseTok}" diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy index a2083eafca9b32..6d9374db849203 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy @@ -107,6 +107,17 @@ suite("test_gram_policy_recovery", "p0") { WHERE msg MATCH_REGEXP '^abcd\$'""") } + // SEARCH TERM and EXACT reach the index as EQUAL, cut by the current (dense4) analyzer, and + // SEARCH has no scalar predicate to fall back on, so it cannot step aside the way MATCH does. + // A gram index refuses SEARCH instead of answering from the recovered dense3 dictionary. + sql "SET enable_inverted_index_query=true" + ["msg:abcdef", "msg:EXACT(abcdef)"].each { dsl -> + test { + sql "SELECT id FROM test_gram_policy_recovery WHERE search('${dsl}')" + exception "gram index" + } + } + // This writer sees dense4, while the recovered rowset retains its dense3 dictionary. // Disable automatic compaction above so both physical schemes remain present together. sql """INSERT INTO test_gram_policy_recovery VALUES From b6a69a3ce1de655bb53d84a18a13e16e2ce72363 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 14 Sep 2026 08:21:07 +0800 Subject: [PATCH 23/36] [fix](be) numeric escapes that Boost reads differently no longer constrain the gram query With enable_extended_regex on, Boost runs every REGEXP pattern that Hyperscan and RE2 both reject, and such a pattern can still get past the gram parser -- an ill-formed UTF-8 byte or a possessive quantifier takes it there. Boost reads numeric escapes differently from the parser's three-digit octal rule, which Hyperscan and RE2 share: \0141 is the octal escape for 'a', taking a fourth digit, and \141 is backreference 1 followed by "41". The parser read \014 then '1', and 'a', and required grams that a row Boost matches does not contain. Recall cases on the real scalar engine lost that row under every dense and sparse scheme, both after an ill-formed byte and in the well-formed pattern (x)ab\141cd++timeout. Which engine runs a pattern is decided outside the index, so the parser no longer relies on it. Only \0dd with no octal digit after it, which all three engines read alike, is still decoded; any other numeric escape is unsupported and the pattern compiles to ALL, on every engine. --- .../storage/index/inverted/gram/regex_ast.cpp | 16 +++-- .../storage/index/inverted/gram/regex_ast.h | 2 +- .../inverted/gram/regex_gram_recall_test.cpp | 61 ++++++++++++++++++- 3 files changed, 70 insertions(+), 9 deletions(-) diff --git a/be/src/storage/index/inverted/gram/regex_ast.cpp b/be/src/storage/index/inverted/gram/regex_ast.cpp index 458e92943a53ee..2e844091266d27 100644 --- a/be/src/storage/index/inverted/gram/regex_ast.cpp +++ b/be/src/storage/index/inverted/gram/regex_ast.cpp @@ -373,10 +373,16 @@ struct Parser { return true; } + static bool is_octal_digit(char ch) { return ch >= '0' && ch <= '7'; } + // Decode only character escapes whose meaning is shared by the scalar engines. Other // letter/digit escapes may denote assertions, classes or backreferences; treating them as - // literals could exclude matching rows. Three-digit octal avoids short numeric escapes' - // ambiguity with backreferences. + // literals could exclude matching rows. A numeric escape is shared in one form only: \0dd + // with no octal digit after it. With extended regex on, Boost runs any pattern Hyperscan and + // RE2 both reject (an ill-formed UTF-8 byte or a possessive quantifier takes one there), and + // Boost reads \0141 as 'a' and \141 as backreference 1, where Hyperscan and RE2 read \014 + // then '1', and 'a'. The engine is chosen outside the index, so every other numeric escape is + // unsupported. bool parse_character_escape(uint32_t* cp) { const char c = p[i++]; switch (c) { @@ -398,9 +404,9 @@ struct Parser { case 'x': return parse_hex_escape_value(cp); default: - if (c >= '0' && c <= '7' && i + 1 < p.size() && p[i] >= '0' && p[i] <= '7' && - p[i + 1] >= '0' && p[i + 1] <= '7') { - *cp = (c - '0') * 64 + (p[i] - '0') * 8 + (p[i + 1] - '0'); + if (c == '0' && i + 1 < p.size() && is_octal_digit(p[i]) && is_octal_digit(p[i + 1]) && + (i + 2 == p.size() || !is_octal_digit(p[i + 2]))) { + *cp = (p[i] - '0') * 8 + (p[i + 1] - '0'); i += 2; return true; } diff --git a/be/src/storage/index/inverted/gram/regex_ast.h b/be/src/storage/index/inverted/gram/regex_ast.h index f2a5696de0921d..642b328978c1c6 100644 --- a/be/src/storage/index/inverted/gram/regex_ast.h +++ b/be/src/storage/index/inverted/gram/regex_ast.h @@ -53,7 +53,7 @@ struct RegexNode { }; // Parse a pattern in the RE2 syntax subset into a RegexNode AST. Supported: literals; escapes -// (`\. \a \f \n \t \r \ooo \xHH \x{...} \Q..\E`, where \ooo is three octal digits); +// (`\. \a \f \n \t \r \0oo \xHH \x{...} \Q..\E`, where \0oo has no octal digit after it); // classes (`[...]`, negation, ranges, POSIX classes, `\d \w \s \v \D \W \S \pL \p{..}`); // `.`; groups (capturing, `(?:`, `(?P`, `(?`); // the flags `(?i) (?s) (?m) (?U)` and `(?i:...)`; the quantifiers `* + ? {m} {m,} {m,n}` and diff --git a/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp b/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp index bd036dd6b3480c..3db7f64ef4f564 100644 --- a/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp +++ b/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp @@ -373,10 +373,17 @@ TEST(RegexGramRecallTest, Re2FallbackEngineMatrix) { {"ab\vcdtimeout", "ab\ncdtimeout", "ab\rcdtimeout", "ab\fcdtimeout", "unrelated"}, {true, false, false, false, false}, ScalarPath::RE2}, + {R"(ab[\061]cdtimeout.{0,51}$)", + {"ab1cdtimeout", "abacdtimeout", "unrelated"}, + {true, false, false}, + ScalarPath::RE2}, + // Only \0dd with no octal digit after it reads alike on every engine, Boost included. {R"(ab[\141]cdtimeout.{0,51}$)", {"abacdtimeout", "ab1cdtimeout", "unrelated"}, {true, false, false}, - ScalarPath::RE2}, + ScalarPath::RE2, + false, + true}, {R"(\Qab😀\E{2}timeout.{0,51}$)", {"ab😀😀timeout", "ab😀timeout", "unrelated"}, {true, false, false}, @@ -719,10 +726,28 @@ TEST(RegexGramRecallTest, EscapesPreserveScalarRecall) { "ab cdtimeout", "ab\tcdtimeout", "abvcdtimeout", "unrelated"}, {true, true, true, true, false, false, false, false}, true); - check_recall(R"(ab\141cdtimeout)", {"abacdtimeout", "ab141cdtimeout", "unrelated"}, + // \0dd with no octal digit after it is the one numeric escape every engine reads alike. + check_recall(R"(ab\061cdtimeout)", {"ab1cdtimeout", "ab061cdtimeout", "unrelated"}, {true, false, false}, true); - check_recall(R"(ab[\141]cdtimeout)", {"abacdtimeout", "ab1cdtimeout", "unrelated"}, + check_recall(R"(ab[\061]cdtimeout)", {"ab1cdtimeout", "ab0cdtimeout", "unrelated"}, {true, false, false}, true); + // Hyperscan reads 'a' and \014 then '1' here, but Boost reads a backreference and a + // four-digit octal escape, so these filter nothing whichever engine runs the pattern. + check_recall(R"(ab\141cdtimeout)", {"abacdtimeout", "ab141cdtimeout", "unrelated"}, + {true, false, false}, /*require_pruning=*/false, + /*enable_hyperscan_fallback=*/true, /*enable_extended_regex=*/false, + ScalarPath::ANY, /*require_all=*/true); + check_recall(R"(ab[\141]cdtimeout)", {"abacdtimeout", "ab1cdtimeout", "unrelated"}, + {true, false, false}, /*require_pruning=*/false, + /*enable_hyperscan_fallback=*/true, /*enable_extended_regex=*/false, + ScalarPath::ANY, /*require_all=*/true); + check_recall(R"(ab\0141cdtimeout)", + {"ab\x0c" + "1cdtimeout", + "abacdtimeout", "unrelated"}, + {true, false, false}, /*require_pruning=*/false, + /*enable_hyperscan_fallback=*/true, /*enable_extended_regex=*/false, + ScalarPath::ANY, /*require_all=*/true); // Hyperscan accepts \Z even though RE2 rejects it. It must not become a literal Z. check_recall(R"(timeout\Z)", {"timeout", "timeoutx", "unrelated"}, {true, false, false}, false); @@ -835,4 +860,34 @@ TEST(RegexGramRecallTest, EmbeddedNulPreservesScalarRecall) { {true, false, false}, true); } +TEST(RegexGramRecallTest, NumericEscapesBoostReadsDifferentlyFilterNothing) { + // With extended regex, Boost runs every pattern that Hyperscan and RE2 both reject, and such + // a pattern can still get past the gram parser. Boost reads \0141 as the octal escape for 'a' + // and \141 as backreference 1 followed by "41", where Hyperscan and RE2 read \014 then '1', + // and 'a'. The row Boost matches must stay a candidate. + const auto check_boost = [](const std::string& pattern, const std::vector& rows, + bool require_pruning) { + check_recall(pattern, rows, {true, false, false}, require_pruning, + /*enable_hyperscan_fallback=*/true, /*enable_extended_regex=*/true, + ScalarPath::BOOST, /*require_all=*/!require_pruning); + }; + // An ill-formed UTF-8 byte is one way onto Boost; it does not stop the gram parser. + const std::string ill_formed = "\xFF"; + check_boost(ill_formed + "ab\\0141cdtimeout", + {ill_formed + "abacdtimeout", + ill_formed + "ab\x0c" + "1cdtimeout", + "unrelated"}, + false); + check_boost(ill_formed + "(x)ab\\141cdtimeout", + {ill_formed + "xabx41cdtimeout", ill_formed + "xabacdtimeout", "unrelated"}, false); + // A possessive quantifier is another, with every byte well-formed. + check_boost("(x)ab\\141cd++timeout", {"xabx41cdtimeout", "xabacdtimeout", "unrelated"}, false); + // \0dd with no octal digit after it names the same code point to Boost, so it still filters. + check_boost(ill_formed + "ab\\061cdtimeout", + {ill_formed + "ab1cdtimeout", ill_formed + "abacdtimeout", "unrelated"}, true); + check_boost(ill_formed + "ab[\\061]cdtimeout", + {ill_formed + "ab1cdtimeout", ill_formed + "abacdtimeout", "unrelated"}, true); +} + } // namespace doris::segment_v2::gram From 0d076bf75df286a0e8c77c8c32f2789a4c19ba7c Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 14 Sep 2026 10:04:55 +0800 Subject: [PATCH 24/36] [chore](be) correct the gram extractor benchmark notes and drop its README The README gave GramScheme's maximum gram length as 16, while the default is 4, and both it and the benchmark comment said fresh=1 followed the writer's per-value tokenizer lifetime. The index writers reuse one tokenizer and its buffers across rows, so fresh=0 is the writer-like case and fresh=1 only measures per-row construction and allocation. The comment now says so and refers to gram_scheme.h for the remaining defaults instead of repeating values that drift. No other benchmark in be/benchmark carries a README, so this one is removed. --- be/benchmark/README_gram_extractor.md | 55 ----------------------- be/benchmark/benchmark_gram_extractor.hpp | 6 ++- 2 files changed, 4 insertions(+), 57 deletions(-) delete mode 100644 be/benchmark/README_gram_extractor.md diff --git a/be/benchmark/README_gram_extractor.md b/be/benchmark/README_gram_extractor.md deleted file mode 100644 index d510eed7c99347..00000000000000 --- a/be/benchmark/README_gram_extractor.md +++ /dev/null @@ -1,55 +0,0 @@ -# Gram extraction benchmark - -`GramExtraction` measures the real `GramExtractor::extract` implementation, including -ASCII folding, boundary selection, per-row deduplication and allocations. It is not a -benchmark of complete token streams, index construction, or SQL queries. - -## Build and run - -Set `BUILD_TYPE=RELEASE` in the local `custom_env.sh` for this performance build, then -use the standard build entry point: - -```bash -./build.sh --benchmark -j "$(nproc)" -./output/be/lib/benchmark_test \ - --benchmark_filter='GramExtraction/' \ - --benchmark_min_time=0.3s \ - --benchmark_repetitions=7 \ - --benchmark_out=gram-extraction.json \ - --benchmark_out_format=json -``` - -On Linux, make the build's `libjvm.so` discoverable through `LD_LIBRARY_PATH` if -needed. Keep correctness tests in the normal ASAN build; do not compare an ASAN -timing with a RELEASE timing. - -## Matrix - -| Argument | Values | -| --- | --- | -| `bytes` | 128, 4096, 65536, 1048576 | -| `sparse` | 0: dense; 1: sparse | -| `lower_case` | 0: preserve ASCII case; 1: fold ASCII case | -| `corpus` | 0: deterministic diverse ASCII; 1: repeated ASCII log; 2: mixed UTF-8 log | -| `fresh` | 1: construct/destroy extractor and output vector per row; 0: reuse both | - -Other parameters use `GramScheme` defaults: minimum length 3, maximum length 16, -density 250/1000, hash version 1. All inputs are generated before timing and reused -as hot input. The `fresh=1` lifetime matches per-value gram tokenization in the -writer; `fresh=0` isolates extraction after constructor and buffer reuse costs. - -`grams_per_row` and `digest` record the initial extraction outside timing. Compare -them before interpreting timing differences between implementations. Correctness -of repeated calls is covered by the gram unit tests, not by a timing threshold. - -For comparisons, use identical compiler flags, dependencies, input and CPU affinity. -Do not compile or run regression workloads concurrently with timing. Run both orders -(for example fixed/old/old/fixed), retain the per-repetition CPU `ns/op`, and inspect -dispersion as well as medians. Extend `benchmark_min_time` for cases with few -iterations or high variability. CPU frequency scaling, SMT and unrelated host load -can still affect measurements. - -Report `fresh` and `reused` separately, as well as corpus and size: constructor -savings on short rows can conceal an extraction-only cost on long rows. A throughput -ratio from this single-threaded, hot-input microbenchmark is not an end-to-end -database speedup or a statement about all tokenizer parameter combinations. diff --git a/be/benchmark/benchmark_gram_extractor.hpp b/be/benchmark/benchmark_gram_extractor.hpp index 5d476a5003071a..8ae560ba387a2a 100644 --- a/be/benchmark/benchmark_gram_extractor.hpp +++ b/be/benchmark/benchmark_gram_extractor.hpp @@ -64,8 +64,10 @@ inline uint64_t gram_digest(const std::vector& grams) { return hash; } -// fresh=1 follows the writer's per-value tokenizer lifetime; fresh=0 isolates -// extraction with reusable buffers. This measures tokenization, not SQL throughput. +// fresh=0 reuses one extractor and its output buffer across rows, as the index writers do; +// fresh=1 constructs both for every row, a synthetic measure of per-row construction and +// allocation cost. Scheme fields other than mode and lower_case keep the GramScheme defaults +// in gram_scheme.h. This measures tokenization, not SQL throughput. inline void GramExtraction(benchmark::State& state) { GramScheme scheme; scheme.mode = state.range(1) == 0 ? GramMode::DENSE : GramMode::SPARSE; From 16bd1d99fc8686732e5b1d0821f95907d4825c08 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 14 Sep 2026 10:04:55 +0800 Subject: [PATCH 25/36] [test](regression) the stop-gram suite no longer changes a cluster-wide BE config The suite runs in the parallel p0 pool, yet it set gram_index_max_candidate_ratio_bp to 0 on every backend and wrote back a fixed 15 afterwards. Overlapping suites could run under a disabled fallback ratio, and a backend that did not use the default lost its setting for the rest of the run. The phase proved nothing either. exceeds_candidate_budget consults the ratio only when the segment has no digest budget, and even then not below gram_index_candidate_ratio_min_rows (65,536 rows). The suite's 4,000-row segment carries a digest, so zeroing the ratio left every answer and every gate decision unchanged. The phase and the config helper are removed; the straddling-pattern profile assertions, which do reach the dropped-posting path, stay. --- .../gram/test_gram_stop_gram.groovy | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy index e0eef13053873b..f342558a584985 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy @@ -55,21 +55,6 @@ suite("test_gram_stop_gram", "p0") { throw new IllegalStateException("analyzer ${name} was not installed on BE", lastNotFound) } - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort) - def set_be_config = { key, value -> - for (String backend_id : backendId_to_backendIP.keySet()) { - def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), - backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update ${key}=${value}: code=${code}, out=${out}, err=${err}") - // A phase that runs against a config that never changed proves nothing. - assertEquals(0, code, "updating ${key} on backend ${backend_id} failed: ${err}") - assertTrue(out.toString().contains("OK"), - "updating ${key} on backend ${backend_id} was refused: ${out}") - } - } - // The tables go first: a policy still referenced by a table left behind by an earlier run // cannot be dropped. sql "DROP TABLE IF EXISTS test_gram_stop_gram_one" @@ -223,20 +208,7 @@ suite("test_gram_stop_gram", "p0") { "the kept posting (df 4) bounds the candidates, got ${candidates}: the dropped " + "grams must read as match-all and the kept one must still be read") assertTrue(pruned >= rows - 4L, "the index pruned only ${pruned} of ${rows} rows") - - // The gate could otherwise mask a broken dropped-gram path by giving up first, so - // check again with its fallback ratio disabled. The gate's primary budget comes from - // the segment itself and cannot be switched off -- that is the point of deriving it - // rather than configuring it -- but zeroing the ratio removes the one arm that a - // configuration could have been hiding behind. - set_be_config("gram_index_max_candidate_ratio_bp", "0") - def ratioOff = runAll(oneSegment, "postings dropped, fallback ratio off") - scanned.each { name, value -> - assertEquals(value[0][0], ratioOff[name][0][0], - "with the fallback ratio disabled the index changed the answer for ${name}") - } } finally { - set_be_config("gram_index_max_candidate_ratio_bp", "15") sql "DROP TABLE IF EXISTS ${oneSegment}" sql "DROP TABLE IF EXISTS ${fiveSegments}" sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_stop_ana" From df1d33eaf5baa3ba90020a616bfd8e8b54fe6161 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 15 Sep 2026 11:01:17 +0800 Subject: [PATCH 26/36] [fix](be) LIKE and REGEXP hand gram index failures to the scan's fallback policy dispatch_query turned every read_from_index failure other than cancellation and memory errors into "no acceleration": a raw IO_ERROR from an SNII read or an INVERTED_INDEX_FILE_NOT_FOUND was logged and the scan silently read every row. The MATCH, IN, comparison and SEARCH push-downs return the status instead, and SegmentIterator::_downgrade_without_index decides: a missing index file follows enable_fallback_on_missing_inverted_index, a corrupted file is downgraded, and anything else fails the scan. LIKE and REGEXP now keep only a declined index (EVALUATE_SKIPPED, NOT_SUPPORTED) local and return every other status. Two SNII reader changes keep that from failing queries the index could never answer: - An index that is not under an analyzer policy declines a gram query before the result cache and before its file is opened, so a column whose only index is a keyword or tokenized one no longer opens that index for LIKE. - A gram query whose logical index is missing from the container (INVERTED_INDEX_SNII_NOT_FOUND, an index never built into the segment) reports INVERTED_INDEX_FILE_NOT_FOUND with the original message, so the fallback session variable governs it. MATCH keeps the container status. gram::may_be_gram_index is the IO-free test for the first case, and resolve_gram_scheme now uses it as well. --- be/src/exprs/function/like.cpp | 14 +--- be/src/exprs/function/like.h | 11 +-- .../index/inverted/gram/gram_family.cpp | 22 +++--- .../storage/index/inverted/gram/gram_family.h | 4 + .../storage/index/snii/snii_index_reader.cpp | 20 ++++- .../exprs/function/like_gram_index_test.cpp | 60 +++++++++++---- .../index/snii/snii_gram_cache_test.cpp | 75 +++++++++++++++++++ 7 files changed, 160 insertions(+), 46 deletions(-) diff --git a/be/src/exprs/function/like.cpp b/be/src/exprs/function/like.cpp index 5252352622b480..de0bc7a546957f 100644 --- a/be/src/exprs/function/like.cpp +++ b/be/src/exprs/function/like.cpp @@ -1111,7 +1111,7 @@ Status FunctionRegexpLike::open(FunctionContext* context, // R8 (unity build): file-scope helpers use a namespace private to this file. namespace like_gram_index_detail { -// Index acceleration may be skipped, but cancellation and memory failures stop the query. +// A declined index leaves the rows to the predicate; the scan decides every other index status. Status dispatch_query(bool is_like, const std::string& pattern, segment_v2::IndexIterator* iter, const IndexFieldNameAndTypePair& data_type_with_name, uint32_t num_rows, segment_v2::InvertedIndexResultBitmap* bitmap_result) { @@ -1126,21 +1126,11 @@ Status dispatch_query(bool is_like, const std::string& pattern, segment_v2::Inde Status query_status = iter->read_from_index(¶m); if (!query_status.ok()) { - if (query_status.is() || - query_status.is() || - query_status.is()) { - return query_status; - } if (query_status.is() || query_status.is()) { return Status::OK(); } - // Every other error only degrades to "no acceleration". LOG_EVERY_N rather than VLOG, - // because this path already means "the index could not be used" and deserves a trace at - // the default log level, while still not flooding the log once per segment. - LOG_EVERY_N(WARNING, 100) << "gram index push-down skipped, read_from_index returned " - << query_status; - return Status::OK(); + return query_status; } segment_v2::InvertedIndexResultBitmap result(param.roaring, nullptr); diff --git a/be/src/exprs/function/like.h b/be/src/exprs/function/like.h index 078acf20c1258e..4168cd09ba937c 100644 --- a/be/src/exprs/function/like.h +++ b/be/src/exprs/function/like.h @@ -390,13 +390,10 @@ class FunctionLikeBase : public IFunction { // Send the original LIKE/REGEXP pattern to the selected reader. The reader compiles it // against its persisted gram scheme and returns an approximate candidate bitmap. // - // Hard semantic constraint (Rulings R26 / R29): the index may only produce a superset of - // candidates, and any index-side failure or inapplicable case may only cost the speedup -- - // all of them simply return OK() without writing bitmap_result, and a problem on the index - // side must never make a LIKE/REGEXP query fail or change its result. This method handles - // a disabled switch, a call shape it cannot compile, a NULL pattern, an unsupported index, - // a compiler result of ALL and index errors. - // The only statuses rethrown are CANCELLED / MEM_LIMIT_EXCEEDED / MEM_ALLOC_FAILED. + // The index only narrows the rows the predicate still checks. A disabled switch, a call shape + // the compiler cannot handle, a NULL pattern and an index that declines the pattern return OK + // without a result. Any other index error is returned, so the scan applies the same fallback + // policy it applies to other index push-downs. enum class GramCompileKind { LIKE, REGEXP }; Status evaluate_gram_index(GramCompileKind kind, const ColumnsWithTypeAndName& arguments, const std::vector& data_type_with_names, diff --git a/be/src/storage/index/inverted/gram/gram_family.cpp b/be/src/storage/index/inverted/gram/gram_family.cpp index f4fadb14ff3c96..d30bcb5b1e95e2 100644 --- a/be/src/storage/index/inverted/gram/gram_family.cpp +++ b/be/src/storage/index/inverted/gram/gram_family.cpp @@ -29,22 +29,24 @@ namespace doris::segment_v2::gram { +bool may_be_gram_index(const std::map& index_properties) { + const std::string name = get_analyzer_name_from_properties(index_properties); + // The same built-in test create_analyzer_provider uses, so both agree on which names are + // built in. + return !name.empty() && !inverted_index::InvertedIndexAnalyzer::is_builtin_analyzer(name); +} + std::optional resolve_gram_scheme( const std::map& index_properties, IndexPolicyMgr* mgr) { - const std::string name = get_analyzer_name_from_properties(index_properties); - // Built-in analyzer names (standard/english/unicode/chinese/icu/...) must short-circuit - // before the policy manager is touched: they are never registered as policies, and - // get_analyzer_provider_by_name would throw "Policy not found" straight away. The test - // reuses InvertedIndexAnalyzer::is_builtin_analyzer -- the same predicate - // create_analyzer_provider uses, so the two sides cannot drift on "what counts as a built-in - // name". - if (name.empty() || inverted_index::InvertedIndexAnalyzer::is_builtin_analyzer(name) || - mgr == nullptr) { + // Built-in names are never registered as policies, so they must not reach the policy + // manager, whose lookup would throw "Policy not found". + if (mgr == nullptr || !may_be_gram_index(index_properties)) { return std::nullopt; } // get_analyzer_provider_by_name throws when the policy is missing (it never returns // nullptr); see the contract in the header. - return mgr->get_analyzer_provider_by_name(name)->gram_scheme(); + return mgr->get_analyzer_provider_by_name(get_analyzer_name_from_properties(index_properties)) + ->gram_scheme(); } } // namespace doris::segment_v2::gram diff --git a/be/src/storage/index/inverted/gram/gram_family.h b/be/src/storage/index/inverted/gram/gram_family.h index a1914904b81019..42e508623ecf64 100644 --- a/be/src/storage/index/inverted/gram/gram_family.h +++ b/be/src/storage/index/inverted/gram/gram_family.h @@ -29,6 +29,10 @@ class IndexPolicyMgr; namespace doris::segment_v2::gram { +// Whether the index properties name an analyzer policy rather than a built-in analyzer or none, +// the only kind of index that can be written as a gram index. It never resolves the policy. +bool may_be_gram_index(const std::map& index_properties); + // Resolve index properties -> analyzer/normalizer name -> policy -> gram scheme, so that a // caller that only has the index properties and cannot reach an analyzer provider (the query // side) can decide whether an index belongs to the "gram family" (an ngram tokenizer carrying a diff --git a/be/src/storage/index/snii/snii_index_reader.cpp b/be/src/storage/index/snii/snii_index_reader.cpp index 1c2f5857c3192e..0b2b857b456b91 100644 --- a/be/src/storage/index/snii/snii_index_reader.cpp +++ b/be/src/storage/index/snii/snii_index_reader.cpp @@ -699,6 +699,13 @@ Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::s // candidate-restricted. context->candidate_rows_consumed = false; const std::string search_str = query_value.get(); + // Only an index under an analyzer policy can have been written as a gram index, so any other + // index declines LIKE and REGEXP before it reads the result cache or opens its file. + if (is_gram_query(query_type) && !gram::may_be_gram_index(_index_meta.properties())) { + return Status::Error( + "index {} has no analyzer policy, so it cannot be a gram index", + _index_meta.index_name()); + } const auto finish_query = [&](const ::doris::snii::reader::LogicalIndexReader* reader) -> Status { if (null_bitmap_cache_handle == nullptr) { @@ -772,8 +779,17 @@ Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::s InvertedIndexCacheHandle searcher_cache_handle; std::unique_ptr<::doris::snii::reader::LogicalIndexReader> uncached_reader; const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; - RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, - &logical_reader)); + if (Status st = _get_logical_reader(context, &searcher_cache_handle, &uncached_reader, + &logical_reader); + !st.ok()) { + // A logical index missing from its container was never built into this segment. A gram + // query reports it as a missing index file, so the scan's policy for missing indexes + // decides whether to evaluate without the index. + if (is_gram_query(query_type) && st.is()) { + return Status::Error("{}", st.msg()); + } + return st; + } // Compare the two optionals, not just two schemes, and whichever side is empty. A segment // written by a legacy ngram tokenizer carries no scheme, and its dictionary holds that // tokenizer's terms, so grams cut by the current analyzer mean nothing there. The reverse diff --git a/be/test/exprs/function/like_gram_index_test.cpp b/be/test/exprs/function/like_gram_index_test.cpp index 333ec6e1a9e590..d45fca1f9cf8e5 100644 --- a/be/test/exprs/function/like_gram_index_test.cpp +++ b/be/test/exprs/function/like_gram_index_test.cpp @@ -126,25 +126,55 @@ TEST(LikeGramIndexTest, ConfigDisabledSkipsPushDown) { EXPECT_FALSE(iterator.queried); } -TEST(LikeGramIndexTest, ArbitraryIndexErrorDegradesToNoResult) { - FunctionRegexpLike function; +TEST(LikeGramIndexTest, DeclinedIndexLeavesTheRowsToThePredicate) { + // An index that cannot answer the pattern declines it, and the rows are left to the predicate. + const std::vector functions {FunctionLike::create(), FunctionRegexpLike::create()}; std::vector names {{"msg", std::make_shared()}}; - ColumnsWithTypeAndName args {const_string_arg("hello|world")}; - const std::vector degradable { - Status::InternalError("boom"), + ColumnsWithTypeAndName args {const_string_arg("hello")}; + const std::vector declined { Status::Error("not supported"), Status::Error("no prunable grams"), - Status::Error("corrupt index image"), - Status::Error("s3 read failed"), }; - for (const auto& injected : degradable) { - RecordingGramIndexIterator iterator; - iterator.query_status = injected; - segment_v2::InvertedIndexResultBitmap result; - const Status status = - function.evaluate_inverted_index(args, names, {&iterator}, 100, nullptr, result); - ASSERT_TRUE(status.ok()) << injected.to_string() << " -> " << status.to_string(); - EXPECT_TRUE(result.is_empty()) << injected.to_string(); + for (const auto& function : functions) { + for (const auto& injected : declined) { + RecordingGramIndexIterator iterator; + iterator.query_status = injected; + segment_v2::InvertedIndexResultBitmap result; + const Status status = function->evaluate_inverted_index(args, names, {&iterator}, 100, + nullptr, result); + ASSERT_TRUE(status.ok()) << function->get_name() << ": " << injected.to_string() + << " -> " << status.to_string(); + EXPECT_TRUE(iterator.queried); + EXPECT_TRUE(result.is_empty()) << injected.to_string(); + } + } +} + +TEST(LikeGramIndexTest, IndexFailuresReachTheSegmentIteratorPolicy) { + // SegmentIterator decides these statuses: FILE_NOT_FOUND follows + // enable_fallback_on_missing_inverted_index, FILE_CORRUPTED is downgraded, and the rest fail + // the scan. The function hands them over unchanged, as the MATCH push-down does. + const std::vector functions {FunctionLike::create(), FunctionRegexpLike::create()}; + std::vector names {{"msg", std::make_shared()}}; + ColumnsWithTypeAndName args {const_string_arg("hello")}; + const std::vector propagated { + ErrorCode::IO_ERROR, + ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND, + ErrorCode::INVERTED_INDEX_FILE_CORRUPTED, + ErrorCode::INVERTED_INDEX_SNII_NOT_FOUND, + ErrorCode::CORRUPTION, + ErrorCode::INTERNAL_ERROR, + }; + for (const auto& function : functions) { + for (const int code : propagated) { + RecordingGramIndexIterator iterator; + iterator.query_status = Status(code, "index failure"); + segment_v2::InvertedIndexResultBitmap result; + const Status status = function->evaluate_inverted_index(args, names, {&iterator}, 100, + nullptr, result); + EXPECT_EQ(status.code(), code) << function->get_name() << ": " << status.to_string(); + EXPECT_TRUE(result.is_empty()); + } } } diff --git a/be/test/storage/index/snii/snii_gram_cache_test.cpp b/be/test/storage/index/snii/snii_gram_cache_test.cpp index a1579f5f7fcc15..2d4e1ad815aec2 100644 --- a/be/test/storage/index/snii/snii_gram_cache_test.cpp +++ b/be/test/storage/index/snii/snii_gram_cache_test.cpp @@ -523,6 +523,81 @@ TEST_F(SniiGramCacheTest, AnalyzedQueriesDeclineWhenTheAnalyzerNoLongerCutsGrams } } +// An index under a built-in analyzer can never have been written as a gram index. It declines LIKE +// and REGEXP before the result cache and before its file is opened, so a file that cannot be read +// never fails a query this index could not answer. +TEST_F(SniiGramCacheTest, NonGramIndexDeclinesGramQueriesWithoutOpeningItsFile) { + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(6753920); + pb.set_index_name("gram_cache_english"); + pb.add_col_unique_id(0); + pb.mutable_properties()->insert({"parser", "english"}); + TabletIndex english; + english.init_from_pb(pb); + const auto missing_file = std::make_shared( + io::global_local_filesystem(), std::string(kTestDir) + "/missing_segment", + InvertedIndexStorageFormatPB::SNII); + const auto english_reader = SniiIndexReader::create_shared( + &english, missing_file, InvertedIndexReaderType::FULLTEXT, _values.size(), + /*column_is_array=*/false); + const auto gram_reader = SniiIndexReader::create_shared( + &_indexes[0], missing_file, InvertedIndexReaderType::FULLTEXT, _values.size(), + /*column_is_array=*/false); + for (const auto type : + {InvertedIndexQueryType::LIKE_GRAM_QUERY, InvertedIndexQueryType::REGEXP_GRAM_QUERY}) { + SCOPED_TRACE(query_type_to_string(type)); + const std::vector declined_request {{english_reader, "abcdef", type}}; + const auto declined = run_queries(declined_request); + EXPECT_TRUE(declined.front().status.is()) + << declined.front().status; + EXPECT_EQ(declined.front().bitmap, nullptr); + EXPECT_EQ(declined.front().stats.inverted_index_query_cache_lookup, 0); + EXPECT_EQ(declined.front().stats.inverted_index_searcher_cache_miss, 0); + + // A gram index over the same missing file still opens it and reports the missing file. + const std::vector gram_request {{gram_reader, "abcdef", type}}; + const auto missing = run_queries(gram_request); + EXPECT_TRUE(missing.front().status.is()) + << missing.front().status; + } +} + +// The container holds no logical index for this index id, so the index was never built into this +// segment. A gram query reports that as a missing index file, which lets +// enable_fallback_on_missing_inverted_index decide the scan; MATCH keeps the container's status. +TEST_F(SniiGramCacheTest, GramQueryReportsALogicalIndexMissingFromItsContainerAsAMissingFile) { + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(6753930); + pb.set_index_name("gram_cache_unbuilt"); + pb.add_col_unique_id(0); + pb.mutable_properties()->insert({"analyzer", "gram_cache_dense3_analyzer"}); + TabletIndex unbuilt; + unbuilt.init_from_pb(pb); + const auto reader = SniiIndexReader::create_shared(&unbuilt, _file_reader, + InvertedIndexReaderType::FULLTEXT, + _values.size(), /*column_is_array=*/false); + for (const auto& [pattern, type] : std::vector> { + {"%abcdef%", InvertedIndexQueryType::LIKE_GRAM_QUERY}, + {"abcdef", InvertedIndexQueryType::REGEXP_GRAM_QUERY}}) { + SCOPED_TRACE(query_type_to_string(type)); + const std::vector requests {{reader, pattern, type}}; + const auto results = run_queries(requests); + const auto& result = results.front(); + EXPECT_TRUE(result.status.is()) << result.status; + EXPECT_NE(result.status.to_string().find("logical index not found"), std::string::npos) + << result.status; + EXPECT_EQ(result.bitmap, nullptr); + } + + const std::vector match_request { + {reader, "abcdef", InvertedIndexQueryType::MATCH_ANY_QUERY}}; + const auto match = run_queries(match_request); + EXPECT_TRUE(match.front().status.is()) + << match.front().status; +} + class SniiGramCacheOptionsTest : public SniiGramCacheTest, public testing::WithParamInterface> {}; From 20a5df3b3b4d61af82fa660418080a06e2fb9496 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 15 Sep 2026 11:01:29 +0800 Subject: [PATCH 27/36] [fix](be) gram parser gives up on regex syntax the scalar engines read differently The gram index may only drop rows the scalar engine cannot match, but the engine is chosen per pattern: Hyperscan when it accepts the pattern, RE2 when Hyperscan rejects or skips it, and Boost when both reject it and enable_extended_regex is on. The parser read some syntax the way only some of these engines do, so the compiled grams pruned rows the chosen engine returns: - Boost skips whitespace inside bounded repeats (a{ 3 }), which Hyperscan and RE2 read as literal text. - RE2 reads a repeat count of ten or more digits, or one with a leading zero (a{1000000000}, a{01}), as literal text, and runs such patterns on the default session. The parser also accumulated counts past INT_MAX in an int, which is undefined behaviour. - Boost reads [.a.] and [=a=] as one collating or equivalence element and closes the class after it, where RE2 closes it at the first ']'. - Boost reads \<, \>, \` and \' as word and buffer anchors, where Hyperscan and RE2 read the escaped character. A brace now becomes a repeat only when every bound is one to nine digits without a leading zero and the braces hold no whitespace. Braces that no engine reads as a repeat stay literal, and any other brace fails the parse, so the pattern compiles to ALL. Collating and equivalence elements, and the four anchor escapes outside a class, fail the parse as well. Checking every escape the parser accepts, at top level and inside a class, against all three engines found no other escape that loses a row. The recall regression suite now compares index-on and index-off counts for these patterns with and without extended regex. --- .../storage/index/inverted/gram/regex_ast.cpp | 115 ++++++-- .../storage/index/inverted/gram/regex_ast.h | 11 +- .../index/inverted/gram/regex_ast_test.cpp | 50 ++++ .../inverted/gram/regex_gram_recall_test.cpp | 265 ++++++++++++++++++ .../gram/test_gram_pattern_recall.groovy | 60 ++++ 5 files changed, 471 insertions(+), 30 deletions(-) diff --git a/be/src/storage/index/inverted/gram/regex_ast.cpp b/be/src/storage/index/inverted/gram/regex_ast.cpp index 2e844091266d27..16acc741bc4f44 100644 --- a/be/src/storage/index/inverted/gram/regex_ast.cpp +++ b/be/src/storage/index/inverted/gram/regex_ast.cpp @@ -259,6 +259,73 @@ struct Parser { } } + // How the scalar engines read a `{` that follows an atom. + enum class BraceReading { LITERAL, REPEAT, AMBIGUOUS }; + + bool skip_repeat_space() { + const size_t start = i; + while (!eof() && std::isspace(static_cast(peek()))) { + i++; + } + return i != start; + } + + // Reads one repeat count at i. Hyperscan, RE2 and Boost all read one to nine digits without a + // leading zero as the same number, while RE2 reads a longer count or a leading zero as literal + // text; those forms return false. The digits are consumed either way and *value cannot + // overflow. + bool parse_repeat_count(int* value) { + const size_t start = i; + int v = 0; + while (!eof() && std::isdigit(static_cast(peek()))) { + if (i - start < 9) { + v = v * 10 + (peek() - '0'); + } + i++; + } + *value = v; + const size_t digits = i - start; + return digits <= 9 && !(digits > 1 && p[start] == '0'); + } + + // Reads `{m}`, `{m,}` or `{m,n}` at i; LITERAL restores i to the brace. Boost skips whitespace + // around the bounds and the comma where Hyperscan and RE2 read literal text, so a spaced form + // is AMBIGUOUS, and so is a count only some engines read as a number. + BraceReading parse_repeat_bounds(int* mn, int* mx) { + const size_t save = i; + i++; // '{' + bool spaced = skip_repeat_space(); + if (eof() || !std::isdigit(static_cast(peek()))) { + i = save; + return BraceReading::LITERAL; + } + bool plain = parse_repeat_count(mn); + if (skip_repeat_space()) { + spaced = true; + } + *mx = *mn; + if (peek() == ',') { + i++; + if (skip_repeat_space()) { + spaced = true; + } + *mx = -1; + if (std::isdigit(static_cast(peek()))) { + const bool upper_plain = parse_repeat_count(mx); + plain = plain && upper_plain; + if (skip_repeat_space()) { + spaced = true; + } + } + } + if (peek() != '}') { + i = save; + return BraceReading::LITERAL; + } + i++; + return plain && !spaced ? BraceReading::REPEAT : BraceReading::AMBIGUOUS; + } + NP parse_quant(NP a) { while (!eof()) { char c = peek(); @@ -278,37 +345,17 @@ struct Parser { s->kids.push_back(std::move(a)); a = std::move(s); } else if (c == '{') { - size_t save = i; - i++; int mn = 0; int mx = -1; - bool has = false; - while (!eof() && std::isdigit(static_cast(peek()))) { - mn = mn * 10 + (peek() - '0'); - i++; - has = true; - } - if (!has) { - i = save; + const BraceReading reading = parse_repeat_bounds(&mn, &mx); + if (reading == BraceReading::LITERAL) { break; } - if (peek() == ',') { - i++; - if (std::isdigit(static_cast(peek()))) { - mx = 0; - while (!eof() && std::isdigit(static_cast(peek()))) { - mx = mx * 10 + (peek() - '0'); - i++; - } - } - } else { - mx = mn; - } - if (peek() != '}') { - i = save; - break; + if (reading == BraceReading::AMBIGUOUS) { + ok = false; + err = "bounded repeat the scalar engines read differently"; + return a; } - i++; NP s = mk(RegexNode::Type::REPEAT); s->rmin = mn; s->rmax = mx; @@ -549,6 +596,13 @@ struct Parser { bool first = true; while (!eof() && (peek() != ']' || first)) { first = false; + if (peek() == '[' && i + 1 < p.size() && (p[i + 1] == '.' || p[i + 1] == '=')) { + // Boost reads [.x.] and [=x=] as one collating or equivalence element and closes + // the class at the bracket after it, while RE2 closes the class at the first ']'. + ok = false; + err = "collating or equivalence element"; + return n; + } if (peek() == '[' && i + 1 < p.size() && p[i + 1] == ':') { // POSIX class, [:alpha:] size_t e = p.find(":]", i); if (e == std::string_view::npos) { @@ -654,6 +708,15 @@ struct Parser { case 'z': i++; return mk(RegexNode::Type::EMPTY); + case '<': + case '>': + case '`': + case '\'': + // Boost reads these as word and buffer anchors, while Hyperscan and RE2 read the + // escaped character. + ok = false; + err = "escape Boost reads as an anchor"; + return nullptr; case 'p': case 'P': { i++; diff --git a/be/src/storage/index/inverted/gram/regex_ast.h b/be/src/storage/index/inverted/gram/regex_ast.h index 642b328978c1c6..24a98bb6a3f4f6 100644 --- a/be/src/storage/index/inverted/gram/regex_ast.h +++ b/be/src/storage/index/inverted/gram/regex_ast.h @@ -53,11 +53,14 @@ struct RegexNode { }; // Parse a pattern in the RE2 syntax subset into a RegexNode AST. Supported: literals; escapes -// (`\. \a \f \n \t \r \0oo \xHH \x{...} \Q..\E`, where \0oo has no octal digit after it); -// classes (`[...]`, negation, ranges, POSIX classes, `\d \w \s \v \D \W \S \pL \p{..}`); +// (`\. \a \f \n \t \r \0oo \xHH \x{...} \Q..\E`, where \0oo has no octal digit after it; outside +// a class the escapes \< \> \` \' are not supported, because Boost reads them as anchors); +// classes (`[...]`, negation, ranges, POSIX classes, `\d \w \s \v \D \W \S \pL \p{..}`, but not +// the collating and equivalence elements `[.x.]` and `[=x=]`); // `.`; groups (capturing, `(?:`, `(?P`, `(?`); -// the flags `(?i) (?s) (?m) (?U)` and `(?i:...)`; the quantifiers `* + ? {m} {m,} {m,n}` and -// their lazy suffixes; the anchors `^ $ \b \B \A \Z \z`; `|`. +// the flags `(?i) (?s) (?m) (?U)` and `(?i:...)`; the quantifiers `* + ? {m} {m,} {m,n}`, each +// bound one to nine digits without a leading zero and no whitespace inside the braces, and their +// lazy suffixes; the anchors `^ $ \b \B \A \Z \z`; `|`. // On success *root owns the whole tree and *case_insensitive says whether `(?i)` appeared in // the pattern; on failure (syntax error, unclosed group/class, dangling escape, group nesting // too deep, ...) Status::InvalidArgument is returned and neither *root nor *case_insensitive is diff --git a/be/test/storage/index/inverted/gram/regex_ast_test.cpp b/be/test/storage/index/inverted/gram/regex_ast_test.cpp index e56c893ce5b3d3..3e1a8608b0aa7b 100644 --- a/be/test/storage/index/inverted/gram/regex_ast_test.cpp +++ b/be/test/storage/index/inverted/gram/regex_ast_test.cpp @@ -172,4 +172,54 @@ TEST(RegexAstTest, HexEscapeHardening) { EXPECT_EQ(parse_dump("\\x{4"), "ERR"); } +// Boost skips whitespace around repeat bounds, while Hyperscan and RE2 read such braces as literal +// text, so a spaced repeat has no reading the gram query can rely on. +TEST(RegexAstTest, BoundedRepeatWithWhitespaceIsRejected) { + for (const char* re : + {"a{ 3 }", "a{3 }", "a{ 3}", "a{\t3}", "a{3, 4}", "a{3 ,4}", "a{ 3 , 4 }", "a{3, }"}) { + EXPECT_EQ(parse_dump(re), "ERR") << re; + } + // Braces that no engine reads as a repeat stay literal text. + EXPECT_EQ(parse_dump("a{ x}"), "cat('a','{',' ','x','}',)"); + EXPECT_EQ(parse_dump("a{3 x}"), "cat('a','{','3',' ','x','}',)"); +} + +// RE2 reads a count of ten or more digits, or one with a leading zero, as literal text, while +// Hyperscan and Boost read a number. +TEST(RegexAstTest, RepeatCountsSomeEngineReadsAsTextAreRejected) { + for (const char* re : {"a{1000000000}", "a{2147483648}", "a{1,4294967297}", "a{0,4294967296}", + "a{01}", "a{1,02}", "a{00}"}) { + EXPECT_EQ(parse_dump(re), "ERR") << re; + } +} + +// Braces every engine reads the same way keep their meaning. +TEST(RegexAstTest, RepeatCountsEveryEngineReadsAlikeKeepTheirShape) { + EXPECT_EQ(parse_dump("a{0}"), "cat(rep('a',0,0),)"); + EXPECT_EQ(parse_dump("a{0,2}"), "cat(rep('a',0,2),)"); + EXPECT_EQ(parse_dump("a{3,}"), "cat(rep('a',3,-1),)"); + EXPECT_EQ(parse_dump("a{2,1000}"), "cat(rep('a',2,1000),)"); + EXPECT_EQ(parse_dump("a{999999999}"), "cat(rep('a',999999999,999999999),)"); + EXPECT_EQ(parse_dump("a{,3}"), "cat('a','{',',','3','}',)"); +} + +// Boost reads [.x.] and [=x=] as one collating or equivalence element and closes the class after +// it, while RE2 closes the class at the first ']'. +TEST(RegexAstTest, CollatingAndEquivalenceElementsAreRejected) { + for (const char* re : {"[[.a.]]x", "[[=a=]]x", "[b[.a.]]x", "[^[=a=]]x", "x[[.space.]]"}) { + EXPECT_EQ(parse_dump(re), "ERR") << re; + } + EXPECT_EQ(parse_dump("[[:alpha:]]x"), "cat([big],'x',)"); + EXPECT_EQ(parse_dump("[[a]x"), "cat([[a],'x',)"); +} + +// At top level Boost reads \< \> \` \' as word and buffer anchors, while Hyperscan and RE2 read +// the escaped character. Inside a class every engine reads the character. +TEST(RegexAstTest, BoostAnchorEscapesAreRejected) { + for (const char* re : {"\\", "\\`x", "x\\'"}) { + EXPECT_EQ(parse_dump(re), "ERR") << re; + } + EXPECT_EQ(parse_dump("[\\<\\>]x"), "cat([<>],'x',)"); +} + } // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp b/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp index 3db7f64ef4f564..29a7856ebfcb5f 100644 --- a/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp +++ b/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp @@ -890,4 +890,269 @@ TEST(RegexGramRecallTest, NumericEscapesBoostReadsDifferentlyFilterNothing) { {ill_formed + "ab1cdtimeout", ill_formed + "abacdtimeout", "unrelated"}, true); } +TEST(RegexGramRecallTest, BoundedRepeatWithWhitespaceFiltersNothing) { + // With extended regex, Boost runs every pattern Hyperscan and RE2 both reject; a possessive + // quantifier is one way there. Boost skips whitespace around the bounds and the comma, so + // `a{ 3 }` repeats `a`, while Hyperscan and RE2 read the same braces as literal text. + const std::vector cases = {{"a{ 3 }timeout++", + {"aaatimeout", "a{ 3 }timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"a{3 }timeout++", + {"aaatimeout", "a{3 }timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"a{ 3}timeout++", + {"aaatimeout", "a{ 3}timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"a{\t3}timeout++", + {"aaatimeout", "a{\t3}timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"a{3, 4}timeout++", + {"aaaatimeout", "a{3, 4}timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"a{3 ,4}timeout++", + {"aaaatimeout", "a{3 ,4}timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"a{ 3 , 4 }timeout++", + {"aaaatimeout", "a{ 3 , 4 }timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"a{3, }timeout++", + {"aaaaatimeout", "a{3, }timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}}; + check_cases(cases, /*fallback=*/true, /*extended=*/true); + + // Hyperscan reads the same braces as literal text, and that row must stay a candidate too. + check_recall("a{ 3 }timeout", {"a{ 3 }timeout", "aaatimeout", "unrelated"}, + {true, false, false}, /*require_pruning=*/false, + /*enable_hyperscan_fallback=*/true, /*enable_extended_regex=*/false, + ScalarPath::HYPERSCAN); + check_recall("a{3, 4}timeout", {"a{3, 4}timeout", "aaaatimeout", "unrelated"}, + {true, false, false}, /*require_pruning=*/false, + /*enable_hyperscan_fallback=*/true, /*enable_extended_regex=*/false, + ScalarPath::HYPERSCAN); +} + +TEST(RegexGramRecallTest, RepeatCountsTheEnginesReadDifferentlyFilterNothing) { + // Boost accepts counts past INT_MAX once Hyperscan and RE2 decline the pattern. Such a count + // must neither overflow in the parser nor constrain the query with a wrapped value. + const std::vector boost_cases = {{"xa{1,4294967297}ytimeout++", + {"xaaytimeout", "xaytimeout", "unrelated"}, + {true, true, false}, + ScalarPath::BOOST, + false, + true}, + {"xa{0,4294967296}ytimeout++", + {"xaytimeout", "xytimeout", "unrelated"}, + {true, true, false}, + ScalarPath::BOOST, + false, + true}, + {"xa{2147483648}ytimeout++", + {"xaytimeout", "unrelated"}, + {false, false}, + ScalarPath::BOOST, + false, + true}, + {"xa{1,2147483648}ytimeout++", + {"xaaytimeout", "xaytimeout", "unrelated"}, + {true, true, false}, + ScalarPath::BOOST, + false, + true}}; + check_cases(boost_cases, /*fallback=*/true, /*extended=*/true); + + // RE2 runs whatever Hyperscan rejects or skips, with or without extended regex, and reads a + // count of ten or more digits, or one with a leading zero, as literal text. Hyperscan and + // Boost read a repeat, so the row holding the literal braces must stay a candidate. + const std::vector re2_cases = { + {"xa{1,4294967297}ytimeout", + {"xa{1,4294967297}ytimeout", "xaytimeout", "unrelated"}, + {true, false, false}, + ScalarPath::RE2, + false, + true}, + {"xa{1000000000}ytimeout", + {"xa{1000000000}ytimeout", "xaytimeout", "unrelated"}, + {true, false, false}, + ScalarPath::RE2, + false, + true}, + {"xa{1,1000000000}ytimeout", + {"xa{1,1000000000}ytimeout", "xaytimeout", "unrelated"}, + {true, false, false}, + ScalarPath::RE2, + false, + true}, + {"xa{01}ytimeout.{0,51}", + {"xa{01}ytimeout", "xaytimeout", "unrelated"}, + {true, false, false}, + ScalarPath::RE2, + false, + true}, + {"xa{1,02}ytimeout.{0,51}", + {"xa{1,02}ytimeout", "xaaytimeout", "unrelated"}, + {true, false, false}, + ScalarPath::RE2, + false, + true}, + {"xa{00}ytimeout.{0,51}", + {"xa{00}ytimeout", "xytimeout", "unrelated"}, + {true, false, false}, + ScalarPath::RE2, + false, + true}}; + for (bool extended : {false, true}) { + check_cases(re2_cases, /*fallback=*/true, extended); + } + + // Hyperscan reads a leading zero as a number, so its row carries no literal braces. + check_recall("xa{01}ytimeout", {"xaytimeout", "xa{01}ytimeout", "unrelated"}, + {true, false, false}, /*require_pruning=*/false, + /*enable_hyperscan_fallback=*/true, /*enable_extended_regex=*/false, + ScalarPath::HYPERSCAN); +} + +TEST(RegexGramRecallTest, RepeatCountsEveryEngineReadsAlikeKeepPruning) { + // Braces every engine reads the same way still constrain the gram query. + const std::vector cases = { + {"ab{0}cdtimeout", + {"acdtimeout", "abcdtimeout", "unrelated"}, + {true, false, false}, + ScalarPath::RE2}, + {"ab{0,2}cdtimeout", + {"acdtimeout", "abbcdtimeout", "abbbcdtimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::HYPERSCAN}, + {"ab{3,}cdtimeout", + {"abbbcdtimeout", "abbbbbcdtimeout", "abbcdtimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::HYPERSCAN}, + {"xa{2,1000}ytimeout", + {"xaaytimeout", "xaytimeout", "unrelated"}, + {true, false, false}, + ScalarPath::RE2}, + {"xa{,3}ytimeout", + {"xa{,3}ytimeout", "xaytimeout", "unrelated"}, + {true, false, false}, + ScalarPath::HYPERSCAN}}; + check_cases(cases, /*fallback=*/true, /*extended=*/false); +} + +TEST(RegexGramRecallTest, CollatingAndEquivalenceElementsFilterNothing) { + // Boost reads [.a.] and [=a=] inside a bracket expression as one element and closes the class + // at the bracket after it. Hyperscan rejects both, and RE2 reads '[', '.' and '=' as members + // and closes the class at the first ']'. + const std::vector boost_cases = {{"[[.a.]]timeout++", + {"atimeout", ".]timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"[[=a=]]timeout++", + {"atimeout", "=]timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"x[[.space.]]timeout++", + {"x timeout", "x.]timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"[b[.a.]]timeout++", + {"atimeout", "btimeout", "unrelated"}, + {true, true, false}, + ScalarPath::BOOST, + false, + true}}; + check_cases(boost_cases, /*fallback=*/true, /*extended=*/true); + + const std::vector re2_cases = {{"[[.a.]]timeout", + {".]timeout", "[]timeout", "atimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::RE2, + false, + true}, + {"[[=a=]]timeout", + {"=]timeout", "[]timeout", "atimeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::RE2, + false, + true}}; + for (bool extended : {false, true}) { + check_cases(re2_cases, /*fallback=*/true, extended); + } +} + +TEST(RegexGramRecallTest, BoostAnchorEscapesFilterNothing) { + // At top level Boost reads \< \> \` \' as word and buffer anchors, while Hyperscan and RE2 read + // the escaped character. A possessive quantifier sends each pattern to Boost. + const std::vector cases = {{"\\ wxyz(?:q)?+", + {"abcd wxyz", "abcd> wxyz", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"abcdwxyz\\>(?:q)?+", + {"abcdwxyz", "abcdwxyz>", "unrelated"}, + {true, true, false}, + ScalarPath::BOOST, + false, + true}, + {"abcdwxyz\\'(?:q)?+", + {"abcdwxyz", "abcdwxyz'", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}}; + check_cases(cases, /*fallback=*/true, /*extended=*/true); + + // Hyperscan reads the escaped character, so that row must stay a candidate as well. + check_recall("\\ + def literal = sqlLiteral(value) + "(${id}, ${literal}, ${literal}, ${literal}, ${literal})" + }.join(',') + sql "INSERT INTO test_gram_pattern_recall_engines VALUES ${engineValues}" + sql "sync" + // With extended regex on, a possessive quantifier sends the pattern to Boost. + def boostPatterns = ['a{ 3 }timeout++', 'a{3, 4}timeout++', 'a{3, }timeout++', + '[[.a.]]timeout++', '[[=a=]]timeout++', 'x[[.space.]]timeout++', + '\\(?:q)?+', "abcdwxyz\\'(?:q)?+", + 'xa{1,4294967297}ytimeout++', 'xa{0,4294967296}ytimeout++'] + // On the default session RE2 runs the patterns Hyperscan rejects or skips. + def re2Patterns = ['xa{1,4294967297}ytimeout', 'xa{1000000000}ytimeout', + 'xa{01}ytimeout.{0,51}', 'xa{1,02}ytimeout.{0,51}', 'xa{00}ytimeout.{0,51}'] + def countRegexp = { String column, String pattern, boolean useIndex -> + sql "SET enable_inverted_index_query=${useIndex}" + return sql("""SELECT count(*) FROM test_gram_pattern_recall_engines + WHERE ${column} REGEXP ${sqlLiteral(pattern)}""")[0][0] + } + def scanned = [:] + def indexed = [:] + [[true, boostPatterns], [false, re2Patterns]].each { extended, engineCases -> + sql "SET enable_extended_regex=${extended}" + schemes.each { column, unused -> + engineCases.each { pattern -> + def label = "extended=${extended} ${column} REGEXP ${pattern}".toString() + scanned[label] = countRegexp(column, pattern, false) + indexed[label] = countRegexp(column, pattern, true) + } + } + } + sql "SET enable_extended_regex=false" + sql "SET enable_inverted_index_query=true" + assertEquals(scanned, indexed, "the gram index lost rows the scalar predicate returns") + scanned.each { label, count -> + assertTrue(count > 0, "${label} matches no row even without the index") + } + // The policies live cluster-wide and the cluster is shared, so a suite that leaves // its own behind eats into the instance-wide policy limit for everyone else. + sql "DROP TABLE IF EXISTS test_gram_pattern_recall_engines" sql "DROP TABLE IF EXISTS test_gram_pattern_recall" schemes.each { name, properties -> sql "DROP INVERTED INDEX ANALYZER IF EXISTS gram_recall_${name}" From a737cc5c2ede8c60773efd3d4e2ef4f865b2e06c Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 15 Sep 2026 11:28:56 +0800 Subject: [PATCH 28/36] [test](regression) the pattern recall suite drops its engine table before the analyzers The engine table added for the scalar-engine parity checks uses the same analyzers as the original table, but it was dropped only at the end of the suite. A run that fails before the end leaves it behind, and the next run then fails at start-up because DROP INVERTED INDEX ANALYZER refuses an analyzer that an index still uses. The suite now drops that table together with the original one before it drops the analyzers. --- .../inverted_index_p0/gram/test_gram_pattern_recall.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy index 8bd5ba3f7fff0f..c5067938935cdf 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy @@ -44,6 +44,8 @@ suite("test_gram_pattern_recall", "p0") { throw new IllegalStateException("analyzer ${name} was not installed on BE", lastNotFound); } + // Both tables use the analyzers below, and a failed run can leave either one behind. + sql "DROP TABLE IF EXISTS test_gram_pattern_recall_engines" sql "DROP TABLE IF EXISTS test_gram_pattern_recall" def schemes = [dense: ["dense", false], sparse: ["sparse", false], dense_lc: ["dense", true], sparse_lc: ["sparse", true]] From 49fb7d6b2b81eebc45844ba4446f81fee086f300 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 15 Sep 2026 11:57:57 +0800 Subject: [PATCH 29/36] [fix](be) gram parser gives up on signed repeat counts that Boost reads as numbers Boost reads a repeat count through an input stream, which accepts a sign directly in front of the digits: a{+3} repeats three times, a{-0} zero times, and a{1,-3} has no upper bound. Hyperscan and RE2 read the same braces as literal text, and so did the gram parser. With extended regex on, a pattern that Hyperscan and RE2 both reject (one with a possessive quantifier, say) runs on Boost, and the grams compiled from the literal reading pruned rows Boost returns. A brace whose lower or upper count carries a sign now fails the parse, so the pattern compiles to ALL. A sign that no digit follows is not part of any count, and those braces stay literal. Checking every printable byte at each position inside a brace, and signs combined with whitespace, against the Hyperscan, RE2 and Boost paths found no other brace reading that loses a row. --- .../storage/index/inverted/gram/regex_ast.cpp | 28 ++++++--- .../index/inverted/gram/regex_ast_test.cpp | 13 ++++ .../inverted/gram/regex_gram_recall_test.cpp | 60 +++++++++++++++++++ .../gram/test_gram_pattern_recall.groovy | 4 +- 4 files changed, 96 insertions(+), 9 deletions(-) diff --git a/be/src/storage/index/inverted/gram/regex_ast.cpp b/be/src/storage/index/inverted/gram/regex_ast.cpp index 16acc741bc4f44..e8cb14acb54d1c 100644 --- a/be/src/storage/index/inverted/gram/regex_ast.cpp +++ b/be/src/storage/index/inverted/gram/regex_ast.cpp @@ -270,10 +270,18 @@ struct Parser { return i != start; } - // Reads one repeat count at i. Hyperscan, RE2 and Boost all read one to nine digits without a - // leading zero as the same number, while RE2 reads a longer count or a leading zero as literal - // text; those forms return false. The digits are consumed either way and *value cannot - // overflow. + // Consumes a sign directly in front of a digit, which Boost reads as part of a repeat count. + bool skip_repeat_sign() { + if ((peek() == '+' || peek() == '-') && i + 1 < p.size() && + std::isdigit(static_cast(p[i + 1]))) { + i++; + return true; + } + return false; + } + + // Reads one repeat count at i and returns false unless it is one to nine digits without a + // leading zero, the only form every engine reads as the same number. bool parse_repeat_count(int* value) { const size_t start = i; int v = 0; @@ -288,13 +296,14 @@ struct Parser { return digits <= 9 && !(digits > 1 && p[start] == '0'); } - // Reads `{m}`, `{m,}` or `{m,n}` at i; LITERAL restores i to the brace. Boost skips whitespace - // around the bounds and the comma where Hyperscan and RE2 read literal text, so a spaced form - // is AMBIGUOUS, and so is a count only some engines read as a number. + // Reads `{m}`, `{m,}` or `{m,n}` at i; LITERAL restores i to the brace. Boost accepts whitespace + // and a sign around the counts where Hyperscan and RE2 read literal text, so those forms are + // AMBIGUOUS, as is a count only some engines read as a number. BraceReading parse_repeat_bounds(int* mn, int* mx) { const size_t save = i; i++; // '{' bool spaced = skip_repeat_space(); + bool signed_count = skip_repeat_sign(); if (eof() || !std::isdigit(static_cast(peek()))) { i = save; return BraceReading::LITERAL; @@ -310,6 +319,9 @@ struct Parser { spaced = true; } *mx = -1; + if (skip_repeat_sign()) { + signed_count = true; + } if (std::isdigit(static_cast(peek()))) { const bool upper_plain = parse_repeat_count(mx); plain = plain && upper_plain; @@ -323,7 +335,7 @@ struct Parser { return BraceReading::LITERAL; } i++; - return plain && !spaced ? BraceReading::REPEAT : BraceReading::AMBIGUOUS; + return plain && !spaced && !signed_count ? BraceReading::REPEAT : BraceReading::AMBIGUOUS; } NP parse_quant(NP a) { diff --git a/be/test/storage/index/inverted/gram/regex_ast_test.cpp b/be/test/storage/index/inverted/gram/regex_ast_test.cpp index 3e1a8608b0aa7b..099a784b8c795b 100644 --- a/be/test/storage/index/inverted/gram/regex_ast_test.cpp +++ b/be/test/storage/index/inverted/gram/regex_ast_test.cpp @@ -222,4 +222,17 @@ TEST(RegexAstTest, BoostAnchorEscapesAreRejected) { EXPECT_EQ(parse_dump("[\\<\\>]x"), "cat([<>],'x',)"); } +// Boost reads a sign directly in front of a repeat count, while Hyperscan and RE2 read the braces +// as literal text, so a signed count has no reading the gram query can rely on. +TEST(RegexAstTest, SignedRepeatCountsAreRejected) { + for (const char* re : {"a{+3}", "a{ +3 }", "a{+3 }", "a{+03}", "a{-0}", "a{-3}", "a{+1,}", + "a{+1,4}", "a{-1,4}", "a{1,+4}", "a{1,-4}", "a{1, -4}"}) { + EXPECT_EQ(parse_dump(re), "ERR") << re; + } + // A sign that no digit follows makes no count for any engine, so the braces stay literal. + EXPECT_EQ(parse_dump("a{+ 3}"), "cat('a',plus('{'),' ','3','}',)"); + EXPECT_EQ(parse_dump("a{3,+}"), "cat('a','{','3',plus(','),'}',)"); + EXPECT_EQ(parse_dump("a{++3}"), "cat('a',plus(plus('{')),'3','}',)"); +} + } // namespace doris::segment_v2::gram diff --git a/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp b/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp index 29a7856ebfcb5f..c83df5419eb8ee 100644 --- a/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp +++ b/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp @@ -1155,4 +1155,64 @@ TEST(RegexGramRecallTest, BoostAnchorEscapesFilterNothing) { /*enable_extended_regex=*/false, ScalarPath::HYPERSCAN); } +TEST(RegexGramRecallTest, SignedRepeatCountsFilterNothing) { + // Boost reads a sign directly in front of a repeat count: a{+3} repeats three times, a{1,-3} + // has no upper bound and a{-0} repeats zero times. A possessive quantifier sends each pattern to + // Boost once extended regex is on. + const std::vector cases = { + {"a{+3}timeout++", + {"aaatimeout", "a{+3}timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"a{ +3 }timeout++", + {"aaatimeout", "a{ +3 }timeout", "unrelated"}, + {true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"a{+1,}timeout++", + {"atimeout", "aaaaaatimeout", "a{+1,}timeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"a{1,+4}timeout++", + {"aaaatimeout", "atimeout", "a{1,+4}timeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"a{1,-3}timeout++", + {"atimeout", "aaaaaatimeout", "a{1,-3}timeout", "unrelated"}, + {true, true, false, false}, + ScalarPath::BOOST, + false, + true}, + {"xa{-0}ytimeout++", + {"xytimeout", "xaytimeout", "xa{-0}ytimeout", "unrelated"}, + {true, false, false, false}, + ScalarPath::BOOST, + false, + true}}; + check_cases(cases, /*fallback=*/true, /*extended=*/true); + + // Hyperscan and RE2 read a signed count as literal text, and a sign that no digit follows is + // literal text to Boost as well, so those braces keep pruning. + check_recall("a{+3}timeout", + {"a{3}timeout", "a{{3}timeout", "aaatimeout", "a{+3}timeout", "unrelated"}, + {true, true, false, false, false}, /*require_pruning=*/false, + /*enable_hyperscan_fallback=*/true, /*enable_extended_regex=*/false, + ScalarPath::HYPERSCAN); + check_recall("a{1,-3}timeout.{0,51}", {"a{1,-3}timeout", "aaatimeout", "unrelated"}, + {true, false, false}, /*require_pruning=*/false, + /*enable_hyperscan_fallback=*/true, /*enable_extended_regex=*/false, + ScalarPath::RE2); + check_recall("a{+ 3}timeout++", {"a{ 3}timeout", "aaatimeout", "unrelated"}, + {true, false, false}, /*require_pruning=*/true, + /*enable_hyperscan_fallback=*/true, /*enable_extended_regex=*/true, + ScalarPath::BOOST); +} + } // namespace doris::segment_v2::gram diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy index c5067938935cdf..cfe90bfd16ee87 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy @@ -158,7 +158,9 @@ suite("test_gram_pattern_recall", "p0") { '[[.a.]]timeout++', '[[=a=]]timeout++', 'x[[.space.]]timeout++', '\\(?:q)?+', "abcdwxyz\\'(?:q)?+", - 'xa{1,4294967297}ytimeout++', 'xa{0,4294967296}ytimeout++'] + 'xa{1,4294967297}ytimeout++', 'xa{0,4294967296}ytimeout++', + 'a{+3}timeout++', 'a{ +3 }timeout++', 'a{+1,}timeout++', + 'a{1,+4}timeout++', 'a{1,-3}timeout++', 'xa{-0}ytimeout++'] // On the default session RE2 runs the patterns Hyperscan rejects or skips. def re2Patterns = ['xa{1,4294967297}ytimeout', 'xa{1000000000}ytimeout', 'xa{01}ytimeout.{0,51}', 'xa{1,02}ytimeout.{0,51}', 'xa{00}ytimeout.{0,51}'] From 566e62fde1aa70f6f10e0ae0fc9ef90c81dce9fa Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 15 Sep 2026 11:58:11 +0800 Subject: [PATCH 30/36] [fix](be) an index under a built-in normalizer declines gram queries without IO may_be_gram_index treated every analyzer or normalizer name other than a built-in analyzer as a possible gram index. A built-in normalizer such as lowercase never cuts grams, yet an index under it still read the result cache and opened its file before declining LIKE and REGEXP. A storage failure while opening that file then failed a query the index could not answer. The check now also rejects the normalizers that IndexPolicyMgr builds in, using the policy manager's own list. --- be/src/exprs/function/like.h | 7 ++-- .../runtime/index_policy/index_policy_mgr.h | 4 +- .../index/inverted/gram/gram_family.cpp | 7 ++-- .../storage/index/inverted/gram/gram_family.h | 5 ++- .../index/snii/snii_gram_cache_test.cpp | 40 +++++++++++++++++++ 5 files changed, 53 insertions(+), 10 deletions(-) diff --git a/be/src/exprs/function/like.h b/be/src/exprs/function/like.h index 4168cd09ba937c..6b56b97a70d602 100644 --- a/be/src/exprs/function/like.h +++ b/be/src/exprs/function/like.h @@ -390,10 +390,9 @@ class FunctionLikeBase : public IFunction { // Send the original LIKE/REGEXP pattern to the selected reader. The reader compiles it // against its persisted gram scheme and returns an approximate candidate bitmap. // - // The index only narrows the rows the predicate still checks. A disabled switch, a call shape - // the compiler cannot handle, a NULL pattern and an index that declines the pattern return OK - // without a result. Any other index error is returned, so the scan applies the same fallback - // policy it applies to other index push-downs. + // Returns OK without a result when the switch is off, the call is not one column and one + // constant pattern, the pattern is NULL, or the index declines it. Any other index error is + // returned so the scan applies its usual fallback policy. enum class GramCompileKind { LIKE, REGEXP }; Status evaluate_gram_index(GramCompileKind kind, const ColumnsWithTypeAndName& arguments, const std::vector& data_type_with_names, diff --git a/be/src/runtime/index_policy/index_policy_mgr.h b/be/src/runtime/index_policy/index_policy_mgr.h index ccdfade6eb3233..c51ca80283ee7f 100644 --- a/be/src/runtime/index_policy/index_policy_mgr.h +++ b/be/src/runtime/index_policy/index_policy_mgr.h @@ -53,6 +53,9 @@ class IndexPolicyMgr { const std::map& outer_char_filter_map = {}, std::string* resolved_name = nullptr, std::string* legacy_name = nullptr); + // Whether the name is one of the normalizers built into Doris rather than a policy. + static bool is_builtin_normalizer(const std::string& name); + private: segment_v2::inverted_index::CustomAnalyzerConfigPtr build_analyzer_config_from_policy( const TIndexPolicy& index_policy_analyzer); @@ -68,7 +71,6 @@ class IndexPolicyMgr { std::function add_config_func); - bool is_builtin_normalizer(const std::string& name); AnalyzerPtr build_builtin_normalizer(const std::string& name); const TIndexPolicy* find_policy_by_name_locked(const std::string& name) const; diff --git a/be/src/storage/index/inverted/gram/gram_family.cpp b/be/src/storage/index/inverted/gram/gram_family.cpp index d30bcb5b1e95e2..725b3c524594c7 100644 --- a/be/src/storage/index/inverted/gram/gram_family.cpp +++ b/be/src/storage/index/inverted/gram/gram_family.cpp @@ -31,9 +31,10 @@ namespace doris::segment_v2::gram { bool may_be_gram_index(const std::map& index_properties) { const std::string name = get_analyzer_name_from_properties(index_properties); - // The same built-in test create_analyzer_provider uses, so both agree on which names are - // built in. - return !name.empty() && !inverted_index::InvertedIndexAnalyzer::is_builtin_analyzer(name); + // The same built-in tests the analyzer and normalizer lookups use, so they agree on which + // names are built in. + return !name.empty() && !inverted_index::InvertedIndexAnalyzer::is_builtin_analyzer(name) && + !IndexPolicyMgr::is_builtin_normalizer(name); } std::optional resolve_gram_scheme( diff --git a/be/src/storage/index/inverted/gram/gram_family.h b/be/src/storage/index/inverted/gram/gram_family.h index 42e508623ecf64..ca027918a41989 100644 --- a/be/src/storage/index/inverted/gram/gram_family.h +++ b/be/src/storage/index/inverted/gram/gram_family.h @@ -29,8 +29,9 @@ class IndexPolicyMgr; namespace doris::segment_v2::gram { -// Whether the index properties name an analyzer policy rather than a built-in analyzer or none, -// the only kind of index that can be written as a gram index. It never resolves the policy. +// Whether the index properties name an analyzer or normalizer policy rather than a built-in +// analyzer, a built-in normalizer or nothing; only such an index can be written as a gram index. +// It never resolves the policy. bool may_be_gram_index(const std::map& index_properties); // Resolve index properties -> analyzer/normalizer name -> policy -> gram scheme, so that a diff --git a/be/test/storage/index/snii/snii_gram_cache_test.cpp b/be/test/storage/index/snii/snii_gram_cache_test.cpp index 2d4e1ad815aec2..85496183305126 100644 --- a/be/test/storage/index/snii/snii_gram_cache_test.cpp +++ b/be/test/storage/index/snii/snii_gram_cache_test.cpp @@ -563,6 +563,46 @@ TEST_F(SniiGramCacheTest, NonGramIndexDeclinesGramQueriesWithoutOpeningItsFile) } } +// A built-in normalizer or a built-in analyzer never cuts grams, so an index under either declines +// LIKE and REGEXP without opening its file, just like an index under a parser. +TEST_F(SniiGramCacheTest, + BuiltinNormalizerOrAnalyzerIndexDeclinesGramQueriesWithoutOpeningItsFile) { + const auto missing_file = std::make_shared( + io::global_local_filesystem(), std::string(kTestDir) + "/missing_segment", + InvertedIndexStorageFormatPB::SNII); + struct BuiltinIndex { + std::string key; + std::string name; + InvertedIndexReaderType reader_type; + }; + for (const auto& builtin : std::vector { + {"normalizer", "lowercase", InvertedIndexReaderType::STRING_TYPE}, + {"analyzer", "english", InvertedIndexReaderType::FULLTEXT}}) { + SCOPED_TRACE(builtin.key + "=" + builtin.name); + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(6753940); + pb.set_index_name("gram_cache_builtin"); + pb.add_col_unique_id(0); + pb.mutable_properties()->insert({builtin.key, builtin.name}); + TabletIndex meta; + meta.init_from_pb(pb); + const auto reader = SniiIndexReader::create_shared(&meta, missing_file, builtin.reader_type, + _values.size(), + /*column_is_array=*/false); + for (const auto type : + {InvertedIndexQueryType::LIKE_GRAM_QUERY, InvertedIndexQueryType::REGEXP_GRAM_QUERY}) { + SCOPED_TRACE(query_type_to_string(type)); + const std::vector requests {{reader, "abcdef", type}}; + const auto results = run_queries(requests); + EXPECT_TRUE(results.front().status.is()) + << results.front().status; + EXPECT_EQ(results.front().stats.inverted_index_query_cache_lookup, 0); + EXPECT_EQ(results.front().stats.inverted_index_searcher_cache_miss, 0); + } + } +} + // The container holds no logical index for this index id, so the index was never built into this // segment. A gram query reports that as a missing index file, which lets // enable_fallback_on_missing_inverted_index decide the scan; MATCH keeps the container's status. From 8951a980726ffcb565f3d7b5f6362bd5bcbb26fa Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 15 Sep 2026 12:23:33 +0800 Subject: [PATCH 31/36] [test](fe) gram-mode ngram tokenizers stay valid with the max_ngram_diff marker Master now stores max_ngram_diff=1 on every new ngram tokenizer and re-validates stored tokenizers whenever an analyzer references them. A gram-mode tokenizer takes its own validation path, which accepts that key and applies neither the size-difference limit nor the absolute size limit. Three tests pin this: the validator accepts mode=sparse with 3/16 and the marker, a replayed gram policy with or without the marker stays valid, and a gram tokenizer created through IndexPolicyMgr keeps the marker and can back an analyzer. The gram branch of the validator gets a comment saying so. --- .../indexpolicy/NGramTokenizerValidator.java | 2 + .../indexpolicy/PolicyValidatorTests.java | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NGramTokenizerValidator.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NGramTokenizerValidator.java index d8fa3e074d4b19..39f80fbccd6d26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NGramTokenizerValidator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NGramTokenizerValidator.java @@ -102,6 +102,8 @@ protected void validateSpecific(Map props) throws DdlException { // The new gram-family parameters: once mode is given this is an auto/sparse/dense gram index, // whose validation rules are entirely independent of legacy ngram, so validate and return // without reusing any of the legacy rules below. + // Gram mode ignores max_ngram_diff, the max_gram - min_gram limit and the absolute size limit, so + // the max_ngram_diff=1 marker that IndexPolicyMgr stores on every new ngram tokenizer stays valid. String mode = props.get("mode"); if (mode != null) { validateGramMode(props, mode); diff --git a/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/PolicyValidatorTests.java b/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/PolicyValidatorTests.java index 3617fb162a9c1c..c898b9c0dcd967 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/PolicyValidatorTests.java +++ b/fe/fe-core/src/test/java/org/apache/doris/indexpolicy/PolicyValidatorTests.java @@ -883,6 +883,61 @@ public void testNGramValidator_GramDecimalPropertiesHavePortableSyntax() { } } + // NGramTokenizerValidator gram mode with the max_ngram_diff compatibility marker + @Test + public void testNGramValidator_GramModeIgnoresDifferenceLimit() { + // IndexPolicyMgr stores max_ngram_diff=1 on every new ngram tokenizer, gram mode included, + // so that marker must not limit max_gram - min_gram once mode is set. + Map props = sparseGramProps(); + props.put("min_gram", "3"); + props.put("max_gram", "16"); + props.put("max_ngram_diff", "1"); + NGramTokenizerValidator validator = new NGramTokenizerValidator(); + Assertions.assertDoesNotThrow(() -> validator.validate(props)); + } + + @Test + public void testGramNGramPolicyWithCompatibilityMarkerRemainsValidAfterReplay() throws Exception { + Map props = sparseGramProps(); + props.put("min_gram", "3"); + props.put("max_gram", "16"); + IndexPolicy withoutMarker = roundTrip(new IndexPolicy( + 3, "gram_without_marker", IndexPolicyTypeEnum.TOKENIZER, props)); + Assertions.assertFalse(withoutMarker.isInvalid()); + + props.put("max_ngram_diff", "1"); + IndexPolicy withMarker = roundTrip(new IndexPolicy( + 4, "gram_with_marker", IndexPolicyTypeEnum.TOKENIZER, props)); + Assertions.assertFalse(withMarker.isInvalid()); + } + + @Test + public void testNewGramNGramPolicyWithCompatibilityMarkerIsUsable() throws Exception { + Map tokenizerProps = sparseGramProps(); + tokenizerProps.put("min_gram", "3"); + tokenizerProps.put("max_gram", "16"); + Map analyzerProps = new HashMap<>(); + analyzerProps.put(IndexPolicy.PROP_TOKENIZER, "new_gram_tokenizer"); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getNextId()).thenReturn(5L, 6L); + Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); + IndexPolicyMgr policyMgr = new IndexPolicyMgr(); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + policyMgr.createIndexPolicy(false, "new_gram_tokenizer", IndexPolicyTypeEnum.TOKENIZER, + tokenizerProps); + // Creating the analyzer re-validates the stored tokenizer, marker included. + policyMgr.createIndexPolicy(false, "new_gram_analyzer", IndexPolicyTypeEnum.ANALYZER, + analyzerProps); + } + + IndexPolicy tokenizer = policyMgr.getPolicyByName("new_gram_tokenizer"); + Assertions.assertEquals("1", tokenizer.getProperties().get("max_ngram_diff")); + Assertions.assertFalse(tokenizer.isInvalid()); + Assertions.assertDoesNotThrow(() -> policyMgr.validateAnalyzerExists("new_gram_analyzer")); + } + // StandardTokenizerValidator Tests @Test public void testStandardTokenizerValidator_ValidProperties() throws Exception { From 383d8a5fd26109cfe5759a0a9856dbba0388248e Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 15 Sep 2026 12:23:52 +0800 Subject: [PATCH 32/36] [chore](be) correct the ngram tokenizer factory comments after the max_ngram_diff change The parse_gram_scheme comment gave the GramScheme defaults as 3/16, but they are 3/4. The initialize comment still described the fixed max-min>1 check that master replaced with configurable size checks. A gram tokenizer test cited factory line numbers that no longer match. The comments now say what the code does without numbers or line references. --- .../tokenizer/ngram/ngram_tokenizer_factory.cpp | 10 +++------- .../index/inverted/tokenizer/gram_tokenizer_test.cpp | 4 +--- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.cpp b/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.cpp index 3c9f7076cb053a..bb95a29301c83b 100644 --- a/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.cpp +++ b/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.cpp @@ -28,12 +28,8 @@ std::unordered_map NGramTokenizerFactory::MATCHERS; Status NGramTokenizerFactory::parse_gram_scheme(const Settings& settings, std::optional* out) { out->reset(); - // The presence of "mode" (sparse|dense|auto) switches to the gram family, which is mutually - // exclusive with the legacy ngram sliding window; parsing, validation and defaults of the - // gram scheme are all delegated to GramScheme::from_properties, and this function only - // forwards the tokenizer properties. Absent min/max_gram come from GramScheme's own member - // initializers (3/16) and are deliberately not duplicated here -- two sets of defaults would - // drift sooner or later, and there can be only one source of truth. + // A "mode" property (sparse, dense or auto) selects the gram family instead of the legacy + // sliding window. GramScheme::from_properties parses, validates and defaults its properties. if (settings.get_string("mode").empty()) { return Status::OK(); } @@ -55,7 +51,7 @@ void NGramTokenizerFactory::initialize(const Settings& settings) { } if (scheme.has_value()) { _gram_scheme = scheme; - return; // skip the legacy max-min>1 validation and token_chars parsing + return; // the legacy size checks and token_chars parsing below do not apply } _min_gram = settings.get_int("min_gram", NGramTokenizer::DEFAULT_MIN_NGRAM_SIZE); diff --git a/be/test/storage/index/inverted/tokenizer/gram_tokenizer_test.cpp b/be/test/storage/index/inverted/tokenizer/gram_tokenizer_test.cpp index 2e9526627059c2..22910de6ad9ada 100644 --- a/be/test/storage/index/inverted/tokenizer/gram_tokenizer_test.cpp +++ b/be/test/storage/index/inverted/tokenizer/gram_tokenizer_test.cpp @@ -128,9 +128,7 @@ TEST(GramTokenizerTest, LowerCaseFoldsBeforeExtraction) { } TEST(GramTokenizerTest, GramModeSkipsLegacyMinMaxGapCheck) { - // Today initialize throws INVALID_ARGUMENT for max_gram-min_gram>1 - // (ngram_tokenizer_factory.cpp:29-36); the gram family returns from inside the mode branch and - // is not subject to that limit. + // The legacy path limits max_gram - min_gram, while the gram family returns before that check. NGramTokenizerFactory factory; std::unordered_map args { {"mode", "sparse"}, {"min_gram", "3"}, {"max_gram", "24"}}; From 0975168adb099a4ab8327bda9a3d6acc89c4c8f3 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 28 Sep 2026 12:06:47 +0800 Subject: [PATCH 33/36] [fix](be) Preserve gram predicate operands and count final candidates ### What problem does this PR solve? Related PR: #67538 Problem Summary: Approximate LIKE and REGEXP push-down lost child roles when it separated index iterators from literal arguments. A reversed predicate could therefore prune matching rows. The gram candidate counter also summed intermediate bitmaps across conjuncts and overstated the final row count. Require an indexed value in the first operand and a literal pattern in the second before gram push-down. Count candidate rows once after all index intersections. Cover reversed operands, a dynamic pattern with ESCAPE, and interleaved approximate and exact conjuncts. ### Release note Gram index push-down preserves correct results for reversed LIKE and REGEXP operands. ### Check List (For Author) - Test - [x] Unit Test: 22 tests in LikeGramBindingTest and SegmentIteratorConjunctShortCircuitTest passed under ASAN. - Behavior changed: - [x] Yes. Reversed or dynamic pattern predicates use scalar evaluation; gram candidate profiling counts the final bitmap once. - Does this need documentation? - [x] No. --- be/src/exprs/vexpr.cpp | 23 ++++---- be/src/storage/segment/segment_iterator.cpp | 24 ++++---- be/src/storage/segment/segment_iterator.h | 6 +- .../exprs/function/like_gram_binding_test.cpp | 57 ++++++++++++++++--- ...t_iterator_conjunct_short_circuit_test.cpp | 28 +++++++++ 5 files changed, 103 insertions(+), 35 deletions(-) diff --git a/be/src/exprs/vexpr.cpp b/be/src/exprs/vexpr.cpp index 46fa77adef29b6..df670fba99411c 100644 --- a/be/src/exprs/vexpr.cpp +++ b/be/src/exprs/vexpr.cpp @@ -919,16 +919,19 @@ ColumnPtr VExpr::get_result_from_const(size_t count) const { Status VExpr::_evaluate_inverted_index(VExprContext* context, const FunctionBasePtr& function, uint32_t segment_num_rows) { - // A function that can only answer approximately (the gram push-down behind LIKE / REGEXP) - // produces a superset of candidate rows, and the caller names the single expression whose - // superset it will read back. Anywhere else -- an operand of a compound AND/OR/NOT, a - // nested child push-down, a virtual column projection -- the bitmap would be stored and - // never read, so building it would spend term dictionary lookups and posting reads (one - // remote request each on object storage) to filter nothing. Bail out before any of that; - // the row-level path still evaluates the predicate exactly, only the speedup is lost. - if (function->index_result_is_approximate() && - context->approx_index_result_consumer() != this) { - return Status::OK(); + if (function->index_result_is_approximate()) { + if (context->approx_index_result_consumer() != this || get_num_children() != 2) { + return Status::OK(); + } + // Keep operand roles intact before the child loop separates iterators from literals. + const auto& value = get_child(0); + const bool indexed_value = + value->is_slot_ref() || + (value->node_type() == TExprNodeType::CAST_EXPR && value->get_num_children() == 1 && + value->get_child(0)->is_slot_ref()); + if (!indexed_value || !get_child(1)->is_literal()) { + return Status::OK(); + } } // Pre-allocate vectors based on an estimated or known size diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index fd23c75554e170..ce02946b3424fc 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -1388,6 +1388,7 @@ Status SegmentIterator::_apply_index_expr() { // iterates the same list. std::vector consumed_by_index; bool bitmap_exhausted = false; + bool applied_approx_index = false; size_t considered_conjuncts = 0; for (const auto& expr_ctx : _common_expr_ctxs_push_down) { if (_row_bitmap.isEmpty()) { @@ -1423,7 +1424,9 @@ Status SegmentIterator::_apply_index_expr() { } else { // Approximate results come here: prune the candidates only, never consume the // expression. - _apply_approx_index_result(expr_ctx.get()); + if (_apply_approx_index_result(expr_ctx.get())) { + applied_approx_index = true; + } } } @@ -1490,6 +1493,10 @@ Status SegmentIterator::_apply_index_expr() { _opts.stats->ann_index_range_cache_hits += ann_index_stats.range_cache_hits.value(); } + if (applied_approx_index) { + _opts.stats->gram_index_candidate_rows += static_cast(_row_bitmap.cardinality()); + } + if (bitmap_exhausted) { // Zero surviving rows satisfy every remaining conjunct, so the whole // list is consumed -- mirroring the column-predicate short circuit. @@ -1509,25 +1516,18 @@ Status SegmentIterator::_apply_index_expr() { return Status::OK(); } -void SegmentIterator::_apply_approx_index_result(VExprContext* expr_ctx) { - // An approximate result is a superset of candidates: rows outside the bitmap certainly do - // not match, while rows inside it may not match. So this only narrows _row_bitmap down to - // the candidate set and never records the expression in consumed_by_index -- it must stay in - // _common_expr_ctxs_push_down so that _execute_common_expr re-verifies each candidate row. +bool SegmentIterator::_apply_approx_index_result(VExprContext* expr_ctx) { + // Approximate results prune candidates while their expressions remain for row evaluation. const auto* approx = expr_ctx->get_index_context()->get_approx_index_result_for_expr(expr_ctx->root().get()); if (approx == nullptr || approx->get_data_bitmap() == nullptr) { - // InvertedIndexResultBitmap allows a null data bitmap (the default-constructed "no - // result" shape). In theory the approximate map only ever holds results with - // is_empty() == false, but dereferencing a null pointer here would be a segfault, which - // costs far more than one extra null check. - return; + return false; } const uint64_t before = _row_bitmap.cardinality(); _row_bitmap &= *approx->get_data_bitmap(); const uint64_t after = _row_bitmap.cardinality(); - _opts.stats->gram_index_candidate_rows += static_cast(after); _opts.stats->rows_gram_index_filtered += static_cast(before - after); + return true; } bool SegmentIterator::_count_on_index_fastpath_safe() const { diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index 978f00be6ad126..5cd4f9e3876505 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -160,10 +160,8 @@ class SegmentIterator : public RowwiseIterator { // below the configured engage ratio; refreshed at conjunct boundaries as // earlier index conjuncts shrink the bitmap. No-op once engaged. void _refresh_candidate_pushdown(); - // An approximate (superset) index result is only used to prune _row_bitmap: intersect it and - // account for the two profile counters, while the expression itself must stay in - // _common_expr_ctxs_push_down to be re-verified by _execute_common_expr. - void _apply_approx_index_result(VExprContext* expr_ctx); + // Intersect an approximate result and keep its expression for row evaluation. + bool _apply_approx_index_result(VExprContext* expr_ctx); // G02: true iff answering the single pushed-down MATCH predicate by its // match COUNT alone is indistinguishable from the row-accurate bitmap for // this COUNT_ON_INDEX scan (no deletes, no other filters, full row bitmap, diff --git a/be/test/exprs/function/like_gram_binding_test.cpp b/be/test/exprs/function/like_gram_binding_test.cpp index 745c4a5efb7ac6..1702cccd31543f 100644 --- a/be/test/exprs/function/like_gram_binding_test.cpp +++ b/be/test/exprs/function/like_gram_binding_test.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -159,7 +160,7 @@ class LikeGramBindingTest : public testing::Test { } Status evaluate_pattern(const std::string& name, const std::string& literal_value, - PatternEvaluation* result) { + PatternEvaluation* result, bool literal_on_left = false) { const DataTypePtr string_type = std::make_shared(); TFunction function; TFunctionName function_name; @@ -179,8 +180,13 @@ class LikeGramBindingTest : public testing::Test { auto slot = VSlotRef::create_shared(0, 0, 0, string_type, "p"); auto literal = VLiteral::create_shared(string_type, Field::create_field(literal_value)); - expression->add_child(slot); - expression->add_child(literal); + if (literal_on_left) { + expression->add_child(literal); + expression->add_child(slot); + } else { + expression->add_child(slot); + expression->add_child(literal); + } std::vector storage_types {{"p", string_type}}; std::unordered_map> index_status; @@ -232,6 +238,7 @@ class LikeGramBindingTest : public testing::Test { // column projection, which only ever materializes exact results) kUnderNot, // NOT(LIKE ...) kUnderOr, // LIKE ... OR LIKE ... + kDynamicPatternWithEscape, }; struct PushDownProbe { @@ -247,29 +254,39 @@ class LikeGramBindingTest : public testing::Test { Status probe_push_down(PushDownShape shape, PushDownProbe* probe) { const DataTypePtr string_type = std::make_shared(); - auto make_like = [&]() { + auto make_like = [&](bool dynamic_pattern_with_escape = false) { TFunction function; TFunctionName function_name; function_name.__set_function_name("like"); function.__set_name(function_name); function.__set_binary_type(TFunctionBinaryType::BUILTIN); - function.__set_arg_types({string_type->to_thrift(), string_type->to_thrift()}); + std::vector arg_types {string_type->to_thrift(), string_type->to_thrift()}; + if (dynamic_pattern_with_escape) { + arg_types.push_back(string_type->to_thrift()); + } + function.__set_arg_types(std::move(arg_types)); function.__set_ret_type(DataTypeUInt8().to_thrift()); function.__set_has_var_args(false); TExprNode node; node.__set_node_type(TExprNodeType::FUNCTION_CALL); node.__set_type(DataTypeUInt8().to_thrift()); node.__set_fn(function); - node.__set_num_children(2); + node.__set_num_children(dynamic_pattern_with_escape ? 3 : 2); node.__set_is_nullable(false); auto call = VectorizedFnCall::create_shared(node); call->add_child(VSlotRef::create_shared(0, 0, 0, string_type, "p")); - call->add_child(VLiteral::create_shared( - string_type, Field::create_field(std::string("%abcdef%")))); + if (dynamic_pattern_with_escape) { + call->add_child(VSlotRef::create_shared(1, 0, 0, string_type, "q")); + call->add_child(VLiteral::create_shared( + string_type, Field::create_field(std::string("!")))); + } else { + call->add_child(VLiteral::create_shared( + string_type, Field::create_field(std::string("%abcdef%")))); + } return call; }; - auto like = make_like(); + auto like = make_like(shape == PushDownShape::kDynamicPatternWithEscape); VExprSPtr root = like; if (shape == PushDownShape::kUnderNot || shape == PushDownShape::kUnderOr) { const bool is_not = shape == PushDownShape::kUnderNot; @@ -360,6 +377,20 @@ TEST_F(LikeGramBindingTest, RegexpPrunesIndexedValues) { check_selective_result(forward); } +TEST_F(LikeGramBindingTest, ReversedOperandsDoNotProduceGramCandidates) { + for (const char* name : {"like", "regexp", "rlike"}) { + SCOPED_TRACE(name); + PatternEvaluation result; + const auto status = evaluate_pattern(name, "abcdef", &result, true); + ASSERT_TRUE(status.ok()) << status; + EXPECT_FALSE(result.has_exact_result); + EXPECT_TRUE(result.candidates.is_empty()); + if (std::string_view(name) == "like") { + EXPECT_TRUE(result.row_result->get_bool(0)); + } + } +} + TEST_F(LikeGramBindingTest, PatternsUseTheSelectedReaderSchemeInASharedContainer) { TIndexPolicy tokenizer; tokenizer.id = 6753822; @@ -442,6 +473,14 @@ TEST_F(LikeGramBindingTest, GramPushDownDoesNoIndexIoWhenItsResultWouldBeDiscard } } +TEST_F(LikeGramBindingTest, DynamicPatternWithEscapeDoesNotReadGramIndex) { + PushDownProbe probe; + const auto status = probe_push_down(PushDownShape::kDynamicPatternWithEscape, &probe); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(probe.index_reader_opens, 0); + EXPECT_FALSE(probe.has_candidates); +} + // An analyzed query (MATCH_*) is only exact on a gram segment when the current analyzer cuts // it with the scheme that cut the segment. Here the physical index was written dense-3, while // the index metadata now names a dense-4 analyzer (a recovered table whose policy was diff --git a/be/test/storage/segment/segment_iterator_conjunct_short_circuit_test.cpp b/be/test/storage/segment/segment_iterator_conjunct_short_circuit_test.cpp index f3d74ca6ecc82e..6f3bdb2a44c9cd 100644 --- a/be/test/storage/segment/segment_iterator_conjunct_short_circuit_test.cpp +++ b/be/test/storage/segment/segment_iterator_conjunct_short_circuit_test.cpp @@ -378,6 +378,34 @@ TEST_F(SegmentIteratorConjunctShortCircuitTest, approx_result_narrows_bitmap_but EXPECT_EQ(_stats.rows_gram_index_filtered, 7); } +TEST_F(SegmentIteratorConjunctShortCircuitTest, approx_candidates_count_final_index_bitmap_once) { + _iter->_row_bitmap.addRange(0, 100); + auto rows_below = [](uint32_t end) { + std::vector rows; + for (uint32_t row = 0; row < end; ++row) { + rows.push_back(row); + } + return rows; + }; + + auto first_approx = std::make_shared(rows_below(80), + BitmapEvalExpr::ResultKind::kApproximate); + auto first_exact = std::make_shared(rows_below(60)); + auto second_approx = std::make_shared(rows_below(50), + BitmapEvalExpr::ResultKind::kApproximate); + auto last_exact = std::make_shared(rows_below(10)); + _iter->_common_expr_ctxs_push_down = { + make_bitmap_ctx(first_approx), make_bitmap_ctx(first_exact), + make_bitmap_ctx(second_approx), make_bitmap_ctx(last_exact)}; + + ASSERT_TRUE(_iter->_apply_index_expr().ok()); + + EXPECT_EQ(_iter->_row_bitmap.cardinality(), 10); + EXPECT_EQ(_iter->_common_expr_ctxs_push_down.size(), 2); + EXPECT_EQ(_stats.gram_index_candidate_rows, 10); + EXPECT_EQ(_stats.rows_gram_index_filtered, 30); +} + // An approximate result that prunes nothing still counts its candidates and reports zero rows // filtered, rather than a negative number from the unsigned difference. TEST_F(SegmentIteratorConjunctShortCircuitTest, approx_result_that_prunes_nothing_filters_none) { From 908fcfe5464d48a2054715455c7a48cd21aefdd6 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 28 Sep 2026 12:20:19 +0800 Subject: [PATCH 34/36] [chore](be) Format SNII integration and simplify gram test setup ### What problem does this PR solve? Related PR: #67538 Problem Summary: The master rebase left two SNII reader and writer lines outside clang-format 16 style. Clang-tidy also found that moving the test argument vector into a Thrift setter taking a const reference had no effect. Apply the formatter and pass the vector directly without changing behavior. ### Release note None ### Check List (For Author) - Test - [x] Unit Test: 22 related ASAN BE tests passed before the redundant move was removed. - [x] Clang-format 16 passed for the changed files. - Behavior changed: - [x] No. - Does this need documentation? - [x] No. --- be/src/storage/index/snii/snii_index_reader.cpp | 10 +++++----- be/src/storage/index/snii/snii_index_writer.cpp | 2 +- be/test/exprs/function/like_gram_binding_test.cpp | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/be/src/storage/index/snii/snii_index_reader.cpp b/be/src/storage/index/snii/snii_index_reader.cpp index 0b2b857b456b91..7c4d3f058b3bde 100644 --- a/be/src/storage/index/snii/snii_index_reader.cpp +++ b/be/src/storage/index/snii/snii_index_reader.cpp @@ -382,7 +382,7 @@ Status execute_snii_query(const ::doris::snii::reader::LogicalIndexReader& logic const InvertedIndexQueryInfo& query_info, std::string_view search_str, const std::vector& terms, int32_t max_expansions, bool collect_phrase_frequency, SniiQueryExecutionResult* result, - ::doris::snii::query::QueryProfile* profile, + ::doris::snii::query::QueryProfile* profile, const roaring::Roaring* candidates, uint64_t rows_of_segment) { result->bitmap = std::make_shared(); result->phrase_matches.clear(); @@ -966,10 +966,10 @@ Status SniiIndexReader::_compute_query_bitmap( query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY); if (needs_prx_profile) { ::doris::snii::SniiPrxExecutionProfileScope execution_profile(*context->stats); - const Status execution_status = - execute_snii_query(*logical_reader, query_type, query_info, search_str, *terms, - max_expansions, phrase_matches != nullptr, &query_result, - execution_profile.profile(), request.candidates, _rows_of_segment); + const Status execution_status = execute_snii_query( + *logical_reader, query_type, query_info, search_str, *terms, max_expansions, + phrase_matches != nullptr, &query_result, execution_profile.profile(), + request.candidates, _rows_of_segment); RETURN_IF_ERROR(execution_status); } else { RETURN_IF_ERROR(execute_snii_query(*logical_reader, query_type, query_info, search_str, diff --git a/be/src/storage/index/snii/snii_index_writer.cpp b/be/src/storage/index/snii/snii_index_writer.cpp index e44b79e986895a..c5d13a2b840e8c 100644 --- a/be/src/storage/index/snii/snii_index_writer.cpp +++ b/be/src/storage/index/snii/snii_index_writer.cpp @@ -29,8 +29,8 @@ #include "common/logging.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/analyzer/analyzer.h" -#include "storage/index/inverted/inverted_index_parser.h" #include "storage/index/inverted/gram/gram_density.h" +#include "storage/index/inverted/inverted_index_parser.h" #include "storage/index/inverted/query/query_info.h" #include "storage/index/inverted/tokenizer/ngram/gram_tokenizer.h" #include "storage/index/snii/query/bm25_scorer.h" diff --git a/be/test/exprs/function/like_gram_binding_test.cpp b/be/test/exprs/function/like_gram_binding_test.cpp index 1702cccd31543f..744b02f59ebc58 100644 --- a/be/test/exprs/function/like_gram_binding_test.cpp +++ b/be/test/exprs/function/like_gram_binding_test.cpp @@ -264,7 +264,7 @@ class LikeGramBindingTest : public testing::Test { if (dynamic_pattern_with_escape) { arg_types.push_back(string_type->to_thrift()); } - function.__set_arg_types(std::move(arg_types)); + function.__set_arg_types(arg_types); function.__set_ret_type(DataTypeUInt8().to_thrift()); function.__set_has_var_args(false); TExprNode node; From fb488b9396a23796b0da0d136dd72c7e70c5be19 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 28 Sep 2026 12:52:14 +0800 Subject: [PATCH 35/36] [test](be) Match dynamic regex fallback expectations ### What problem does this PR solve? Related PR: #67538 Problem Summary: The gram recall test asserted that dynamic REGEXP patterns never use Boost.Regex. The scalar path calls compile_regex with enable_extended_regex, so RE2-rejected patterns can use Boost when that option is enabled. This made six assertions fail even though the scalar and vector results agreed. Name the dynamic path for the selected scalar function and check both valid Boost fallback patterns and syntax rejected by both engines. Assert the returned matches when fallback succeeds. ### Release note None ### Check List (For Author) - Test - [x] Unit Test: The old DynamicPatternRe2Execution case failed in the 54-test ASAN run. - [x] Unit Test: DynamicPatternRegexExecution and FunctionLikeTest.regexp_extended_regex_all_pattern_modes passed under ASAN after the correction. - Behavior changed: - [x] No. Test expectations now match the existing scalar regex behavior. - Does this need documentation? - [x] No. --- .../inverted/gram/regex_gram_recall_test.cpp | 57 ++++++++++++------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp b/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp index c83df5419eb8ee..b8d404ec9f0ea6 100644 --- a/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp +++ b/be/test/storage/index/inverted/gram/regex_gram_recall_test.cpp @@ -43,7 +43,7 @@ enum class ScalarPath { ANY, HYPERSCAN, RE2, - DYNAMIC_RE2, + DYNAMIC_REGEX, BOOST, ALLPASS, EQUALS, @@ -74,7 +74,7 @@ void check_scalar_path(const LikeState& state, ScalarPath path) { case ScalarPath::BOOST: EXPECT_EQ(*selected, &FunctionLikeBase::constant_regex_fn); break; - case ScalarPath::DYNAMIC_RE2: { + case ScalarPath::DYNAMIC_REGEX: { using ScalarFunctionPointer = Status (*)(const LikeSearchState*, const StringRef&, const StringRef&, unsigned char*); const auto* scalar = state.scalar_function.target(); @@ -124,7 +124,7 @@ Status scalar_matches(const std::string& pattern, const std::vector values->insert_data(row.data(), row.size()); } auto patterns = ColumnString::create(); - const bool dynamic_pattern = path == ScalarPath::DYNAMIC_RE2; + const bool dynamic_pattern = path == ScalarPath::DYNAMIC_REGEX; const size_t pattern_rows = dynamic_pattern ? rows.size() : 1; for (size_t i = 0; i < pattern_rows; ++i) { patterns->insert_data(pattern.data(), pattern.size()); @@ -630,62 +630,77 @@ TEST(RegexGramRecallTest, InvalidPatternsRejectAcrossEngineOptions) { } } -TEST(RegexGramRecallTest, DynamicPatternRe2Execution) { - // A ColumnString pattern reaches vector_non_const -> scalar_function -> RE2 even - // with Hyperscan fallback disabled. Extended regex does not enable Boost on this path. +TEST(RegexGramRecallTest, DynamicPatternRegexExecution) { + // A ColumnString pattern reaches the scalar regex function. Extended regex allows + // Boost when RE2 rejects a pattern, even with Hyperscan fallback disabled. const std::vector regexp_cases = { {R"(ab\vcdtimeout)", {"ab\vcdtimeout", "ab\ncdtimeout", "ab\rcdtimeout", "unrelated"}, {true, false, false, false}, - ScalarPath::DYNAMIC_RE2}, + ScalarPath::DYNAMIC_REGEX}, {"(?i:ask)(?-i:timeout)", {"ASKtimeout", "aſktimeout", "asKtimeout", "ASKTIMEOUT", "unrelated"}, {true, true, true, false, false}, - ScalarPath::DYNAMIC_RE2}, + ScalarPath::DYNAMIC_REGEX}, {R"(\Qab😀\E{2}timeout)", {"ab😀😀timeout", "ab😀timeout", "unrelated"}, {true, false, false}, - ScalarPath::DYNAMIC_RE2}, + ScalarPath::DYNAMIC_REGEX}, {std::string("foo[0-9]") + '\0' + "timeout", {std::string("foo1") + '\0' + "timeout", "foo1", "foo1timeout", "unrelated"}, {true, false, false, false}, - ScalarPath::DYNAMIC_RE2, + ScalarPath::DYNAMIC_REGEX, false, true}}; const std::vector like_cases = { {"prefix_timeout", {"prefix😀timeout", "prefix\ntimeout", "prefixtimeout", "unrelated"}, {true, true, false, false}, - ScalarPath::DYNAMIC_RE2}, + ScalarPath::DYNAMIC_REGEX}, {"%prefix%\ntimeout%", {"prefix\ntimeout", "prefixx\ntimeout", "prefixxtimeout", "unrelated"}, {true, true, false, false}, - ScalarPath::DYNAMIC_RE2}, + ScalarPath::DYNAMIC_REGEX}, {R"(prefix\%timeout_)", {"prefix%timeoutX", "prefixxtimeoutX", "unrelated"}, {true, false, false}, - ScalarPath::DYNAMIC_RE2}, + ScalarPath::DYNAMIC_REGEX}, {"étimeout_", {"étimeout😀", "Étimeout😀", "unrelated"}, {true, false, false}, - ScalarPath::DYNAMIC_RE2}, + ScalarPath::DYNAMIC_REGEX}, {std::string("prefix_") + '\0' + "%timeout%", {std::string("prefix1") + '\0' + "timeout", "prefix1", "prefix1timeout", "unrelated"}, {true, false, false, false}, - ScalarPath::DYNAMIC_RE2, + ScalarPath::DYNAMIC_REGEX, false, true}}; for (bool fallback : {false, true}) { for (bool extended : {false, true}) { check_cases(regexp_cases, fallback, extended); check_cases(like_cases, fallback, extended); - for (const auto* pattern : - {R"(timeout\Z)", R"(ab\hcdtimeout)", R"(timeout\xZZ)", "timeout(?=END)"}) { - SCOPED_TRACE(pattern); + struct FallbackCase { + const char* pattern; + bool boost_valid; + std::vector expected; + }; + const std::vector fallback_cases = { + {R"(timeout\Z)", true, {false, false}}, + {R"(ab\hcdtimeout)", true, {false, false}}, + {R"(timeout\xZZ)", false, {}}, + {"timeout(?=END)", true, {true, false}}}; + for (const auto& test_case : fallback_cases) { + SCOPED_TRACE(test_case.pattern); std::vector matches; - auto status = scalar_matches(pattern, {"timeoutEND", "unrelated"}, &matches, - fallback, extended, ScalarPath::DYNAMIC_RE2); - EXPECT_FALSE(status.ok()) << "Dynamic patterns do not use Hyperscan or Boost"; + auto status = + scalar_matches(test_case.pattern, {"timeoutEND", "unrelated"}, &matches, + fallback, extended, ScalarPath::DYNAMIC_REGEX); + if (extended && test_case.boost_valid) { + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(matches, test_case.expected); + } else { + EXPECT_FALSE(status.ok()); + } } } } From e2d34cbc4f852ca018653a55bbbad11fd0f58962 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 28 Sep 2026 15:28:03 +0800 Subject: [PATCH 36/36] [test](regression) Correct empty regex golden results ### What problem does this PR solve? Related PR: #67538 Problem Summary: The new gram regression golden expected an empty regular expression to match no rows. Doris REGEXP and RLIKE match every non-null string for this pattern, so the P0 suite failed at regexp_23_idx_true before it could finish. Regenerate the golden through the regression runner. Both index modes now expect the same matches for REGEXP, RLIKE, NOT REGEXP, and the compound predicate. ### Release note None ### Check List (For Author) - Test - [x] Regression test: test_gram_regexp_like passed with generated output and again with normal golden comparison. - [x] Unit Test: FunctionLikeTest.regexp_empty_constant_pattern passed under ASAN. - Behavior changed: - [x] No. Only the expected regression output changed. - Does this need documentation? - [x] No. --- .../gram/test_gram_regexp_like.out | 74 ++++++++++++++++++- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/regression-test/data/inverted_index_p0/gram/test_gram_regexp_like.out b/regression-test/data/inverted_index_p0/gram/test_gram_regexp_like.out index ebfeceaf6292e8..8dbba98cafc5ee 100644 --- a/regression-test/data/inverted_index_p0/gram/test_gram_regexp_like.out +++ b/regression-test/data/inverted_index_p0/gram/test_gram_regexp_like.out @@ -699,10 +699,26 @@ 8 -- !regexp_23_idx_true -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 -- !rlike_23_idx_true -- - --- !notregexp_23_idx_true -- 1 10 11 @@ -722,7 +738,24 @@ 8 9 +-- !notregexp_23_idx_true -- + -- !regexp_and_23_idx_true -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +4 +6 +7 +8 +9 -- !regexp_24_idx_true -- 14 @@ -1843,10 +1876,26 @@ 8 -- !regexp_23_idx_false -- +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +6 +7 +8 +9 -- !rlike_23_idx_false -- - --- !notregexp_23_idx_false -- 1 10 11 @@ -1866,7 +1915,24 @@ 8 9 +-- !notregexp_23_idx_false -- + -- !regexp_and_23_idx_false -- +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +4 +6 +7 +8 +9 -- !regexp_24_idx_false -- 14