Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,14 @@
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.Like;
import org.apache.doris.nereids.trees.expressions.Match;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SearchExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
import org.apache.doris.nereids.trees.expressions.functions.scalar.MultiMatch;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Search;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.trees.plans.Plan;
Expand Down Expand Up @@ -210,7 +214,9 @@ public static Set<Expression> infer(Set<Expression> inputs) {
}
Map<Expression, Set<Expression>> exprPredicates = new HashMap<>();
for (Expression input : inputs) {
if (input.anyMatch(expr -> !((ExpressionTrait) expr).isDeterministic())
// MATCH, multi_match and SEARCH depend on column indexes, and copies can change score().
if (input.containsType(Match.class, MultiMatch.class, Search.class, SearchExpression.class)
|| input.anyMatch(expr -> !((ExpressionTrait) expr).isDeterministic())
|| input.getInputSlots().size() != 1) {
continue;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,18 @@
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.Like;
import org.apache.doris.nereids.trees.expressions.MatchAny;
import org.apache.doris.nereids.trees.expressions.MatchPhrase;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SearchExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Abs;
import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
import org.apache.doris.nereids.trees.expressions.functions.scalar.MultiMatch;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Search;
import org.apache.doris.nereids.trees.expressions.functions.scalar.SearchDslParser;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
Expand All @@ -47,11 +54,63 @@
import com.google.common.collect.ImmutableList;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;

import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.stream.Stream;

public class InferPredicateByReplaceTest {
@ParameterizedTest
@MethodSource("matchAndSearchPredicates")
public void testDoNotInferMatchOrSearchPredicates(Expression predicate, SlotReference a, SlotReference b) {
Set<Expression> inputs = new LinkedHashSet<>(ImmutableList.of(new EqualTo(a, b), predicate));

Assertions.assertEquals(inputs, InferPredicateByReplace.infer(inputs));
Assertions.assertEquals(inputs, PredicateInferUtils.inferPredicate(inputs));
Assertions.assertEquals(inputs, PredicateInferUtils.inferAllPredicate(inputs));
}

private static Stream<Arguments> matchAndSearchPredicates() {
SlotReference a = new SlotReference("a", StringType.INSTANCE);
SlotReference b = new SlotReference("b", StringType.INSTANCE);
return Stream.of(
new MatchAny(a, new StringLiteral("apple")),
new MatchPhrase(a, new StringLiteral("apple red")),
new MultiMatch(a, new StringLiteral("any"), new StringLiteral("apple")),
new Search(new StringLiteral("a:apple")),
new SearchExpression("a:apple", SearchDslParser.parseDsl("a:apple"), ImmutableList.of(a)))
.flatMap(search -> {
Expression or = new Or(search, new EqualTo(a, new StringLiteral("pear")));
return Stream.of(or,
new EqualTo(search, BooleanLiteral.TRUE),
new EqualTo(or, BooleanLiteral.TRUE),
new GreaterThan(new Cast(or, IntegerType.INSTANCE), new IntegerLiteral(0)),
new InPredicate(or, ImmutableList.of(BooleanLiteral.TRUE)),
new Not(new EqualTo(or, BooleanLiteral.TRUE)),
new Like(new Cast(or, StringType.INSTANCE), new StringLiteral("%true%")));
})
.map(predicate -> Arguments.of(predicate, a, b));
}

@Test
public void testInferOrdinaryPredicatesAlongsideMatch() {
SlotReference a = new SlotReference("a", StringType.INSTANCE);
SlotReference b = new SlotReference("b", StringType.INSTANCE);
Expression match = new Or(new MatchAny(a, new StringLiteral("apple")),
new MatchAny(a, new StringLiteral("pear")));
Set<Expression> inputs = new LinkedHashSet<>(ImmutableList.of(
new EqualTo(a, b), match, new Like(a, new StringLiteral("apple%"))));
Set<Expression> expected = new LinkedHashSet<>(inputs);
expected.add(new Like(b, new StringLiteral("apple%")));

Assertions.assertEquals(expected, InferPredicateByReplace.infer(inputs));
Assertions.assertEquals(expected, PredicateInferUtils.inferAllPredicate(inputs));
}

@Test
public void testInferWithEqualTo() {
SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !different_index --
1

-- !wrapped_match --
1

-- !search --
1

-- !multi_match --
1

-- !score --
1 0.4700036
2 0.4700036

-- !minimum_score --

Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

// An equality between two columns does not let a MATCH on one column be copied to the other:
// MATCH depends on the inverted index of its column, and a copy would add to score().
suite("infer_match_predicate") {
// c2 has an index without a parser, so MATCH on c2 only matches a whole value.
sql "DROP TABLE IF EXISTS infer_match_predicate_parser"
sql """
CREATE TABLE infer_match_predicate_parser (
id INT,
c1 TEXT,
c2 TEXT,
INDEX idx_c1 (c1) USING INVERTED PROPERTIES ("parser" = "english"),
INDEX idx_c2 (c2) USING INVERTED
) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES ("replication_num" = "1")
"""
sql "INSERT INTO infer_match_predicate_parser VALUES (1, 'apple red', 'apple red'), (2, 'banana', 'banana')"
order_qt_different_index """
SELECT id FROM infer_match_predicate_parser
WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear')
"""
order_qt_wrapped_match """
SELECT id FROM infer_match_predicate_parser
WHERE c1 = c2 AND CAST((c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear') AS INT) > 0
"""
order_qt_search """
SELECT id FROM infer_match_predicate_parser
WHERE c1 = c2 AND (search('c1:apple') OR c1 = 'pear')
"""
order_qt_multi_match """
SELECT id FROM infer_match_predicate_parser
WHERE c1 = c2 AND (multi_match(c1, 'any', 'apple') OR multi_match(c1, 'any', 'pear'))
"""

sql "DROP TABLE IF EXISTS infer_match_predicate_score"
sql """
CREATE TABLE infer_match_predicate_score (
id INT,
c1 TEXT,
c2 TEXT,
INDEX idx_c1 (c1) USING INVERTED PROPERTIES ("parser" = "english"),
INDEX idx_c2 (c2) USING INVERTED PROPERTIES ("parser" = "english")
) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES ("replication_num" = "1")
"""
sql """
INSERT INTO infer_match_predicate_score VALUES
(1, 'apple red', 'apple red'), (2, 'apple green', 'apple green'), (3, 'banana yellow', 'banana yellow')
"""
order_qt_score """
SELECT id, score() AS s FROM infer_match_predicate_score
WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear') ORDER BY s DESC LIMIT 10
"""
order_qt_minimum_score """
SELECT id, score() AS s FROM infer_match_predicate_score
WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear') AND score() > 0.6
ORDER BY s DESC LIMIT 10
"""
}
Loading