From 8889a7d2faaf8b68345aa6da483beadef8a8f573 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Sat, 24 Oct 2026 03:13:37 +0000 Subject: [PATCH] [fix](nereids) Do not infer MATCH and SEARCH predicates across column equalities ### What problem does this PR solve? Related PR: #69419 Problem Summary: Backport #69419 to branch-4.2. Predicate inference copied a predicate on one column to every column that is equal to it, also when the predicate contained MATCH, multi_match or SEARCH. Their result depends on the inverted index of the column, not only on its value: with c1 = c2 and different parsers on the two columns, the copy removed matching rows, and with the same index it doubled score(). InferPredicateByReplace.infer now skips every input that contains Match, MultiMatch, Search or SearchExpression. The branch does not have the NoneMovableFunction check that master has next to the new check, so only the new check is added. The test imports SearchDslParser from its package on the branch. ### Release note Fix wrong results and scores when predicate inference copied a MATCH, multi_match or SEARCH predicate to an equal column. ### Check List (For Author) - Test: Includes the unit and regression tests of #69419; results are recorded in the pull request. - Behavior changed: Yes. Predicate inference no longer copies a predicate that contains MATCH, multi_match or SEARCH to an equal column. - Does this need documentation: No. Source-PR: https://github.com/apache/doris/pull/69419 Source-Head: 9e661c801d690f36711fdf148379ec2959ab9a15 Co-Authored-By: Claude Opus 5.5 --- .../rewrite/InferPredicateByReplace.java | 8 +- .../rewrite/InferPredicateByReplaceTest.java | 59 +++++++++++++++ .../infer_predicate/infer_match_predicate.out | 19 +++++ .../infer_match_predicate.groovy | 75 +++++++++++++++++++ 4 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/nereids_rules_p0/infer_predicate/infer_match_predicate.out create mode 100644 regression-test/suites/nereids_rules_p0/infer_predicate/infer_match_predicate.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java index c2ca99f0b61b9c..f5cd0a185184fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java @@ -28,10 +28,14 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.Like; +import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; +import org.apache.doris.nereids.trees.expressions.functions.scalar.MultiMatch; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Search; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.trees.plans.Plan; @@ -210,7 +214,9 @@ public static Set infer(Set inputs) { } Map> exprPredicates = new HashMap<>(); for (Expression input : inputs) { - if (input.anyMatch(expr -> !((ExpressionTrait) expr).isDeterministic()) + // MATCH, multi_match and SEARCH depend on column indexes, and copies can change score(). + if (input.containsType(Match.class, MultiMatch.class, Search.class, SearchExpression.class) + || input.anyMatch(expr -> !((ExpressionTrait) expr).isDeterministic()) || input.getInputSlots().size() != 1) { continue; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java index 87f88157a0f348..2fec983dd21ecd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java @@ -24,11 +24,18 @@ import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.Like; +import org.apache.doris.nereids.trees.expressions.MatchAny; +import org.apache.doris.nereids.trees.expressions.MatchPhrase; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.scalar.Abs; import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.functions.scalar.MultiMatch; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Search; +import org.apache.doris.nereids.trees.expressions.functions.scalar.SearchDslParser; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; @@ -47,11 +54,63 @@ import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Set; +import java.util.stream.Stream; public class InferPredicateByReplaceTest { + @ParameterizedTest + @MethodSource("matchAndSearchPredicates") + public void testDoNotInferMatchOrSearchPredicates(Expression predicate, SlotReference a, SlotReference b) { + Set inputs = new LinkedHashSet<>(ImmutableList.of(new EqualTo(a, b), predicate)); + + Assertions.assertEquals(inputs, InferPredicateByReplace.infer(inputs)); + Assertions.assertEquals(inputs, PredicateInferUtils.inferPredicate(inputs)); + Assertions.assertEquals(inputs, PredicateInferUtils.inferAllPredicate(inputs)); + } + + private static Stream matchAndSearchPredicates() { + SlotReference a = new SlotReference("a", StringType.INSTANCE); + SlotReference b = new SlotReference("b", StringType.INSTANCE); + return Stream.of( + new MatchAny(a, new StringLiteral("apple")), + new MatchPhrase(a, new StringLiteral("apple red")), + new MultiMatch(a, new StringLiteral("any"), new StringLiteral("apple")), + new Search(new StringLiteral("a:apple")), + new SearchExpression("a:apple", SearchDslParser.parseDsl("a:apple"), ImmutableList.of(a))) + .flatMap(search -> { + Expression or = new Or(search, new EqualTo(a, new StringLiteral("pear"))); + return Stream.of(or, + new EqualTo(search, BooleanLiteral.TRUE), + new EqualTo(or, BooleanLiteral.TRUE), + new GreaterThan(new Cast(or, IntegerType.INSTANCE), new IntegerLiteral(0)), + new InPredicate(or, ImmutableList.of(BooleanLiteral.TRUE)), + new Not(new EqualTo(or, BooleanLiteral.TRUE)), + new Like(new Cast(or, StringType.INSTANCE), new StringLiteral("%true%"))); + }) + .map(predicate -> Arguments.of(predicate, a, b)); + } + + @Test + public void testInferOrdinaryPredicatesAlongsideMatch() { + SlotReference a = new SlotReference("a", StringType.INSTANCE); + SlotReference b = new SlotReference("b", StringType.INSTANCE); + Expression match = new Or(new MatchAny(a, new StringLiteral("apple")), + new MatchAny(a, new StringLiteral("pear"))); + Set inputs = new LinkedHashSet<>(ImmutableList.of( + new EqualTo(a, b), match, new Like(a, new StringLiteral("apple%")))); + Set expected = new LinkedHashSet<>(inputs); + expected.add(new Like(b, new StringLiteral("apple%"))); + + Assertions.assertEquals(expected, InferPredicateByReplace.infer(inputs)); + Assertions.assertEquals(expected, PredicateInferUtils.inferAllPredicate(inputs)); + } + @Test public void testInferWithEqualTo() { SlotReference a = new SlotReference("a", IntegerType.INSTANCE); diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/infer_match_predicate.out b/regression-test/data/nereids_rules_p0/infer_predicate/infer_match_predicate.out new file mode 100644 index 00000000000000..cdc045bdd81794 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/infer_predicate/infer_match_predicate.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !different_index -- +1 + +-- !wrapped_match -- +1 + +-- !search -- +1 + +-- !multi_match -- +1 + +-- !score -- +1 0.4700036 +2 0.4700036 + +-- !minimum_score -- + diff --git a/regression-test/suites/nereids_rules_p0/infer_predicate/infer_match_predicate.groovy b/regression-test/suites/nereids_rules_p0/infer_predicate/infer_match_predicate.groovy new file mode 100644 index 00000000000000..6cdb841dae86a6 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/infer_predicate/infer_match_predicate.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// An equality between two columns does not let a MATCH on one column be copied to the other: +// MATCH depends on the inverted index of its column, and a copy would add to score(). +suite("infer_match_predicate") { + // c2 has an index without a parser, so MATCH on c2 only matches a whole value. + sql "DROP TABLE IF EXISTS infer_match_predicate_parser" + sql """ + CREATE TABLE infer_match_predicate_parser ( + id INT, + c1 TEXT, + c2 TEXT, + INDEX idx_c1 (c1) USING INVERTED PROPERTIES ("parser" = "english"), + INDEX idx_c2 (c2) USING INVERTED + ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql "INSERT INTO infer_match_predicate_parser VALUES (1, 'apple red', 'apple red'), (2, 'banana', 'banana')" + order_qt_different_index """ + SELECT id FROM infer_match_predicate_parser + WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear') + """ + order_qt_wrapped_match """ + SELECT id FROM infer_match_predicate_parser + WHERE c1 = c2 AND CAST((c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear') AS INT) > 0 + """ + order_qt_search """ + SELECT id FROM infer_match_predicate_parser + WHERE c1 = c2 AND (search('c1:apple') OR c1 = 'pear') + """ + order_qt_multi_match """ + SELECT id FROM infer_match_predicate_parser + WHERE c1 = c2 AND (multi_match(c1, 'any', 'apple') OR multi_match(c1, 'any', 'pear')) + """ + + sql "DROP TABLE IF EXISTS infer_match_predicate_score" + sql """ + CREATE TABLE infer_match_predicate_score ( + id INT, + c1 TEXT, + c2 TEXT, + INDEX idx_c1 (c1) USING INVERTED PROPERTIES ("parser" = "english"), + INDEX idx_c2 (c2) USING INVERTED PROPERTIES ("parser" = "english") + ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql """ + INSERT INTO infer_match_predicate_score VALUES + (1, 'apple red', 'apple red'), (2, 'apple green', 'apple green'), (3, 'banana yellow', 'banana yellow') + """ + order_qt_score """ + SELECT id, score() AS s FROM infer_match_predicate_score + WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear') ORDER BY s DESC LIMIT 10 + """ + order_qt_minimum_score """ + SELECT id, score() AS s FROM infer_match_predicate_score + WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear') AND score() > 0.6 + ORDER BY s DESC LIMIT 10 + """ +}