Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,9 @@ public class OrExpansion extends DefaultPlanRewriter<OrExpandsionContext> implem
.Builder<JoinType>()
.add(JoinType.INNER_JOIN)
.add(JoinType.LEFT_ANTI_JOIN)
.add(JoinType.RIGHT_ANTI_JOIN)
.add(JoinType.LEFT_OUTER_JOIN)
.add(JoinType.RIGHT_OUTER_JOIN)
.add(JoinType.FULL_OUTER_JOIN)
.build();

Expand Down Expand Up @@ -160,19 +162,23 @@ public Plan visitLogicalJoin(LogicalJoin<? extends Plan, ? extends Plan> join, O
joins.addAll(expandInnerJoin(ctx.cascadesContext, hashOtherConditions,
join, leftProducer, rightProducer, leftCloneToLeft, rightCloneToRight));
} else if (join.getJoinType().isOuterJoin()) {
// left outer join = inner join union left anti join
// Outer join = matching rows union unmatched rows from each preserved side.
joins.addAll(expandInnerJoin(ctx.cascadesContext, hashOtherConditions,
join, leftProducer, rightProducer, leftCloneToLeft, rightCloneToRight));
joins.add(expandLeftAntiJoin(ctx.cascadesContext,
hashOtherConditions, join, leftProducer, rightProducer, leftCloneToLeft, rightCloneToRight));
if (join.getJoinType().equals(JoinType.FULL_OUTER_JOIN)) {
// full outer join = inner join union left anti join union right anti join
if (join.getJoinType().isLeftOuterJoin() || join.getJoinType().isFullOuterJoin()) {
joins.add(expandLeftAntiJoin(ctx.cascadesContext,
hashOtherConditions, join, leftProducer, rightProducer, leftCloneToLeft, rightCloneToRight));
}
if (join.getJoinType().isRightOuterJoin() || join.getJoinType().isFullOuterJoin()) {
joins.add(expandLeftAntiJoin(ctx.cascadesContext, hashOtherConditions,
join, rightProducer, leftProducer, rightCloneToRight, leftCloneToLeft));
}
} else if (join.getJoinType().equals(JoinType.LEFT_ANTI_JOIN)) {
joins.add(expandLeftAntiJoin(ctx.cascadesContext, hashOtherConditions,
join, leftProducer, rightProducer, leftCloneToLeft, rightCloneToRight));
} else if (join.getJoinType().isRightAntiJoin()) {
joins.add(expandLeftAntiJoin(ctx.cascadesContext, hashOtherConditions,
join, rightProducer, leftProducer, rightCloneToRight, leftCloneToLeft));
} else {
throw new RuntimeException("or-expansion is not supported for " + join);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,25 @@

package org.apache.doris.nereids.rules.rewrite;

import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.utframe.TestWithFeService;

import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;

import java.util.List;
import java.util.stream.Collectors;

class OrExpansionTest extends TestWithFeService implements MemoPatternMatchSupported {
@Override
Expand Down Expand Up @@ -63,15 +73,16 @@ void testOrExpand() {
Assertions.assertTrue(plan.child(1) instanceof LogicalCTEAnchor);
}

@Test
void testOrExpandCTE() {
@ParameterizedTest
@EnumSource(value = JoinType.class, names = {"INNER_JOIN", "RIGHT_OUTER_JOIN", "RIGHT_ANTI_JOIN"})
void testOrExpandCTE(JoinType joinType) {
connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
connectContext.getSessionVariable().inlineCTEReferencedThreshold = 0;
String sql = "with t3 as (select t1.id1 + 1 as id1, t1.id2 + 2 as id2 from t1), "
+ "t4 as (select t2.id1 + 1 as id1, t2.id2 + 2 as id2 from t2) "
+ "select t3.id1 from "
+ "select t4.id1 from "
+ "(select id1, id2 from t3 group by id1, id2) t3 "
+ " join "
+ " " + joinType.name().replace('_', ' ') + " "
+ "(select id1, id2 from t4 group by id1, id2) t4 "
+ "on t3.id1 = t4.id1 or t3.id2 = t4.id2";
Plan plan = PlanChecker.from(connectContext)
Expand All @@ -87,4 +98,58 @@ void testOrExpandCTE() {
Assertions.assertTrue(plan.child(1).child(1).child(1)
.anyMatch(x -> x instanceof LogicalCTEConsumer));
}

@ParameterizedTest
@EnumSource(value = JoinType.class, names = {
"INNER_JOIN", "LEFT_OUTER_JOIN", "RIGHT_OUTER_JOIN", "FULL_OUTER_JOIN",
"LEFT_ANTI_JOIN", "RIGHT_ANTI_JOIN"})
void testExpansionPreservesOutput(JoinType joinType) {
String sql = "select * from t1 " + joinType.name().replace('_', ' ') + " t2"
+ " on t1.id1 = t2.id1 or t1.id2 = t2.id2";
PlanChecker checker = PlanChecker.from(connectContext).analyze(sql);
List<Slot> output = checker.getPlan().getOutput();
Plan plan = checker.applyCustom(OrExpansion.INSTANCE).getPlan();
Assertions.assertEquals(output, plan.getOutput());
Assertions.assertEquals(output.stream().map(Slot::nullable).collect(Collectors.toList()),
plan.getOutput().stream().map(Slot::nullable).collect(Collectors.toList()));
List<LogicalUnion> unions = plan.collectToList(LogicalUnion.class::isInstance);
Assertions.assertEquals(1, unions.size(), plan.treeString());
int innerBranches = joinType.isInnerJoin() || joinType.isOuterJoin() ? 2 : 0;
int antiBranches = joinType.isFullOuterJoin() ? 2 : (joinType.isInnerJoin() ? 0 : 1);
Assertions.assertEquals(innerBranches + antiBranches, unions.get(0).arity(), plan.treeString());
}

@ParameterizedTest
@EnumSource(value = JoinType.class, names = {"RIGHT_OUTER_JOIN", "RIGHT_ANTI_JOIN"})
void testRightExpansionPreservesRightInput(JoinType joinType) {
String sql = "select * from t1 " + joinType.name().replace('_', ' ') + " t2"
+ " on (t1.id1 + 1 = t2.id1 or t1.id2 = t2.id2 + 1 or t2.id1 + 2 = t1.id2)"
+ " and t1.id1 < t2.id2";
Plan plan = PlanChecker.from(connectContext).analyze(sql).applyCustom(OrExpansion.INSTANCE).getPlan();
List<LogicalCTEProducer> producers = plan.collectToList(LogicalCTEProducer.class::isInstance);
LogicalUnion union = (LogicalUnion) plan.collectToList(LogicalUnion.class::isInstance).get(0);
Plan unmatched = union.child(union.arity() - 1);
List<LogicalCTEConsumer> consumers = unmatched.collectToList(LogicalCTEConsumer.class::isInstance);
// The preserved right input is read once; each OR disjunct removes matches from it.
Assertions.assertEquals(1, consumers.stream()
.filter(c -> c.getCteId().equals(producers.get(1).getCteId())).count(), plan.treeString());
Assertions.assertEquals(3, consumers.stream()
.filter(c -> c.getCteId().equals(producers.get(0).getCteId())).count(), plan.treeString());
List<LogicalJoin> antiJoins = unmatched.collectToList(LogicalJoin.class::isInstance);
Assertions.assertEquals(3, antiJoins.size(), plan.treeString());
for (LogicalJoin<?, ?> antiJoin : antiJoins) {
Assertions.assertEquals(JoinType.LEFT_ANTI_JOIN, antiJoin.getJoinType());
Assertions.assertEquals(1, antiJoin.getHashJoinConjuncts().size());
Assertions.assertEquals(1, antiJoin.getOtherJoinConjuncts().size());
}
}

@ParameterizedTest
@EnumSource(value = JoinType.class, names = {"RIGHT_OUTER_JOIN", "RIGHT_ANTI_JOIN"})
void testRightExpansionRequiresHashDisjuncts(JoinType joinType) {
String sql = "select * from t1 " + joinType.name().replace('_', ' ') + " t2"
+ " on t1.id1 = t2.id1 or t1.id2 > t2.id2";
Plan plan = PlanChecker.from(connectContext).analyze(sql).applyCustom(OrExpansion.INSTANCE).getPlan();
Assertions.assertFalse(plan.anyMatch(LogicalUnion.class::isInstance), plan.treeString());
}
}
223 changes: 223 additions & 0 deletions regression-test/data/query_p0/union/or_expansion_right_join.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,223 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !enabled --
\N \N \N 105 \N \N 5
\N \N \N 106 7 70 6
\N \N \N 106 7 70 6
1 1 10 101 1 10 1
2 2 20 102 9 20 2
2 2 20 102 9 20 2
3 \N 30 103 \N 30 3
4 4 \N 104 4 \N 4

-- !disabled --
\N \N \N 105 \N \N 5
\N \N \N 106 7 70 6
\N \N \N 106 7 70 6
1 1 10 101 1 10 1
2 2 20 102 9 20 2
2 2 20 102 9 20 2
3 \N 30 103 \N 30 3
4 4 \N 104 4 \N 4

-- !enabled --
\N \N \N 103 \N 30 3
\N \N \N 104 4 \N 4
\N \N \N 105 \N \N 5
\N \N \N 106 7 70 6
\N \N \N 106 7 70 6
1 1 10 101 1 10 1
2 2 20 102 9 20 2
2 2 20 102 9 20 2

-- !disabled --
\N \N \N 103 \N 30 3
\N \N \N 104 4 \N 4
\N \N \N 105 \N \N 5
\N \N \N 106 7 70 6
\N \N \N 106 7 70 6
1 1 10 101 1 10 1
2 2 20 102 9 20 2
2 2 20 102 9 20 2

-- !enabled --
\N \N \N 101 1 10 1
\N \N \N 102 9 20 2
\N \N \N 103 \N 30 3
\N \N \N 104 4 \N 4
\N \N \N 105 \N \N 5
6 6 60 106 7 70 6
6 6 60 106 7 70 6

-- !disabled --
\N \N \N 101 1 10 1
\N \N \N 102 9 20 2
\N \N \N 103 \N 30 3
\N \N \N 104 4 \N 4
\N \N \N 105 \N \N 5
6 6 60 106 7 70 6
6 6 60 106 7 70 6

-- !enabled --
\N \N \N 106 7 70 6
\N \N \N 106 7 70 6
1 1 10 101 1 10 1
2 2 20 102 9 20 2
2 2 20 102 9 20 2
3 \N 30 103 \N 30 3
3 \N 30 105 \N \N 5
4 4 \N 104 4 \N 4
5 \N \N 103 \N 30 3
5 \N \N 105 \N \N 5

-- !disabled --
\N \N \N 106 7 70 6
\N \N \N 106 7 70 6
1 1 10 101 1 10 1
2 2 20 102 9 20 2
2 2 20 102 9 20 2
3 \N 30 103 \N 30 3
3 \N 30 105 \N \N 5
4 4 \N 104 4 \N 4
5 \N \N 103 \N 30 3
5 \N \N 105 \N \N 5

-- !case_enabled --
\N \N \N 102 9 20 2
\N \N \N 103 \N 30 3
\N \N \N 104 4 \N 4
\N \N \N 105 \N \N 5
\N \N \N 106 7 70 6
\N \N \N 106 7 70 6
1 1 10 101 1 10 1

-- !case_disabled --
\N \N \N 102 9 20 2
\N \N \N 103 \N 30 3
\N \N \N 104 4 \N 4
\N \N \N 105 \N \N 5
\N \N \N 106 7 70 6
\N \N \N 106 7 70 6
1 1 10 101 1 10 1

-- !enabled --
105 \N \N 5
106 7 70 6
106 7 70 6

-- !disabled --
105 \N \N 5
106 7 70 6
106 7 70 6

-- !enabled --
103 \N 30 3
104 4 \N 4
105 \N \N 5
106 7 70 6
106 7 70 6

-- !disabled --
103 \N 30 3
104 4 \N 4
105 \N \N 5
106 7 70 6
106 7 70 6

-- !enabled --
101 1 10 1
102 9 20 2
103 \N 30 3
104 4 \N 4
105 \N \N 5

-- !disabled --
101 1 10 1
102 9 20 2
103 \N 30 3
104 4 \N 4
105 \N \N 5

-- !enabled --
106 7 70 6
106 7 70 6

-- !disabled --
106 7 70 6
106 7 70 6

-- !case_enabled --
102 9 20 2
103 \N 30 3
104 4 \N 4
105 \N \N 5
106 7 70 6
106 7 70 6

-- !case_disabled --
102 9 20 2
103 \N 30 3
104 4 \N 4
105 \N \N 5
106 7 70 6
106 7 70 6

-- !aggregate --
101 1 10 1
102 2 40 4
103 1 30 3
104 1 \N 4
105 0 \N 5
106 0 \N 12

-- !aggregate_disabled --
101 1 10 1
102 2 40 4
103 1 30 3
104 1 \N 4
105 0 \N 5
106 0 \N 12

-- !empty_enabled --
\N \N \N 101 1 10 1
\N \N \N 102 9 20 2
\N \N \N 103 \N 30 3
\N \N \N 104 4 \N 4
\N \N \N 105 \N \N 5
\N \N \N 106 7 70 6
\N \N \N 106 7 70 6

-- !empty_disabled --
\N \N \N 101 1 10 1
\N \N \N 102 9 20 2
\N \N \N 103 \N 30 3
\N \N \N 104 4 \N 4
\N \N \N 105 \N \N 5
\N \N \N 106 7 70 6
\N \N \N 106 7 70 6

-- !empty_enabled --

-- !empty_disabled --

-- !empty_enabled --
101 1 10 1
102 9 20 2
103 \N 30 3
104 4 \N 4
105 \N \N 5
106 7 70 6
106 7 70 6

-- !empty_disabled --
101 1 10 1
102 9 20 2
103 \N 30 3
104 4 \N 4
105 \N \N 5
106 7 70 6
106 7 70 6

-- !empty_enabled --

-- !empty_disabled --

Loading
Loading