diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopN.java index b225bb891e3eae..387354112d2287 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopN.java @@ -174,9 +174,10 @@ private static void collectFromNode(Plan node, PullUpInfo info, Set bloc if (node instanceof LogicalProject) { LogicalProject project = (LogicalProject) node; Set childBlockedExprIds = new HashSet<>(blockedExprIds); + Set repeatedIdentitySourceExprIds = collectRepeatedIdentitySourceExprIds(project.getProjects()); for (NamedExpression ne : project.getProjects()) { info.addPullUpExprReplace(ne); - boolean canPullUp = canPullUp(ne); + boolean canPullUp = canPullUp(ne, repeatedIdentitySourceExprIds); if (canPullUp && !blockedExprIds.contains(ne.getExprId())) { info.addPulledUpExpr(project, ne); } @@ -275,14 +276,17 @@ private static void collectFromNode(Plan node, PullUpInfo info, Set bloc /** * Check if a named expression can be pulled up above TopN. - * Eligible: Alias with non-trivial child, not blocked, no NoneMovableFunction. + * Eligible: repeated Alias(Slot), or Alias with a non-trivial movable child. */ - static boolean canPullUp(NamedExpression ne) { + static boolean canPullUp(NamedExpression ne, Set repeatedIdentitySourceExprIds) { if (!(ne instanceof Alias)) { return false; } Expression child = ((Alias) ne).child(); - if (child instanceof Slot || child instanceof Literal) { + if (child instanceof Slot) { + return repeatedIdentitySourceExprIds.contains(((Slot) child).getExprId()); + } + if (child instanceof Literal) { return false; } if (ne.anyMatch(e -> e instanceof NoneMovableFunction)) { @@ -300,6 +304,25 @@ static boolean canPullUp(NamedExpression ne) { return true; } + private static Set collectRepeatedIdentitySourceExprIds(List projects) { + Set identitySourceExprIds = new HashSet<>(); + Set repeatedIdentitySourceExprIds = new HashSet<>(); + for (NamedExpression project : projects) { + ExprId sourceExprId; + if (project instanceof Slot) { + sourceExprId = project.getExprId(); + } else if (project instanceof Alias && project.child(0) instanceof Slot) { + sourceExprId = ((Slot) project.child(0)).getExprId(); + } else { + continue; + } + if (!identitySourceExprIds.add(sourceExprId)) { + repeatedIdentitySourceExprIds.add(sourceExprId); + } + } + return repeatedIdentitySourceExprIds; + } + private static boolean shouldBlockProjectInputs( NamedExpression ne, boolean canPullUp, Set blockedExprIds) { if (blockedExprIds.contains(ne.getExprId())) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java index ee0942c30716d9..a8adec53584b4d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java @@ -142,6 +142,38 @@ public void testLazyBaseColumnIndexUsesOriginalColumnNameForAlias() throws Excep materializeNodes.get(0).getLazyBaseColumnIndices()); } + @Test + public void testDuplicateAliasesShareOneLazyFetchSlot() throws Exception { + this.createTable("create table lazy_materialize_duplicate_alias_tbl(" + + "sort_col int, lazy_col int) " + + "duplicate key(sort_col) distributed by hash(sort_col) buckets 1 " + + "properties('replication_num' = '1')"); + String sql = "select lazy_col as first_alias, lazy_col as second_alias " + + "from lazy_materialize_duplicate_alias_tbl order by sort_col limit 1"; + + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .implement(); + PhysicalPlan plan = checker.getPhysicalPlan(); + plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan); + + List> materializeNodes = plan.collectToList( + node -> node instanceof PhysicalLazyMaterialize); + Assertions.assertEquals(1, materializeNodes.size(), plan.treeString()); + PhysicalLazyMaterialize materialize = materializeNodes.get(0); + Assertions.assertEquals(1, materialize.getRelations().size()); + Assertions.assertEquals(1, materialize.getLazySlots(materialize.getRelations().get(0)).size()); + Assertions.assertEquals(ImmutableList.of(ImmutableList.of(1)), materialize.getLazyBaseColumnIndices()); + + Assertions.assertEquals(2, plan.getOutput().size()); + Assertions.assertEquals("first_alias", plan.getOutput().get(0).getName()); + Assertions.assertEquals("second_alias", plan.getOutput().get(1).getName()); + Assertions.assertNotEquals(plan.getOutput().get(0).getExprId(), plan.getOutput().get(1).getExprId()); + Assertions.assertEquals(plan.getOutput().get(0).getDataType(), plan.getOutput().get(1).getDataType()); + Assertions.assertEquals(plan.getOutput().get(0).nullable(), plan.getOutput().get(1).nullable()); + } + @Test public void testLightSchemaChangeFalse() throws Exception { this.createTable("create table tm_lsc_false (k int, v int) duplicate key(k) " diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java index 692f978e9f0c11..996819ce67cb7c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java @@ -48,6 +48,7 @@ import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; @@ -107,6 +108,116 @@ void testNotPullUpSimpleAlias() { .getPlan(); } + @Test + void testPullUpTwoAndThreeAliasesOfSameSlot() { + assertRepeatedIdentityAliasesPulledUp(2); + assertRepeatedIdentityAliasesPulledUp(3); + } + + @Test + void testPullUpDirectSlotAndRepeatedAliases() { + Slot id = scan1.getOutput().get(0); + Slot source = scan1.getOutput().get(1); + Alias first = source.alias("first"); + Alias second = source.alias("second"); + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .projectExprs(ImmutableList.of(source, first, second, id)) + .topN(3, 0, ImmutableList.of(3)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .getPlan(); + + LogicalProject upperProject = (LogicalProject) rewritten; + Assertions.assertEquals(4, upperProject.getProjects().size()); + Assertions.assertEquals(source.getExprId(), upperProject.getProjects().get(0).getExprId()); + Assertions.assertEquals(first.getExprId(), upperProject.getProjects().get(1).getExprId()); + Assertions.assertEquals(second.getExprId(), upperProject.getProjects().get(2).getExprId()); + Assertions.assertEquals(id.getExprId(), upperProject.getProjects().get(3).getExprId()); + + LogicalTopN topN = (LogicalTopN) upperProject.child(0); + LogicalProject lowerProject = (LogicalProject) topN.child(0); + Assertions.assertEquals(2, lowerProject.getProjects().size()); + Assertions.assertEquals(source.getExprId(), lowerProject.getProjects().get(0).getExprId()); + Assertions.assertEquals(id.getExprId(), lowerProject.getProjects().get(1).getExprId()); + } + + @Test + void testPullUpRepeatedAliasesThroughForwardingProject() { + Slot id = scan1.getOutput().get(0); + Slot source = scan1.getOutput().get(1); + Alias first = source.alias("first"); + Alias second = source.alias("second"); + LogicalProject lowerProject = new LogicalProject<>( + ImmutableList.of(first, second, id), scan1); + LogicalProject> forwardingProject = new LogicalProject<>( + ImmutableList.of(first.toSlot(), second.toSlot(), id), lowerProject); + LogicalPlan plan = new LogicalPlanBuilder(forwardingProject) + .topN(3, 0, ImmutableList.of(2)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .getPlan(); + + LogicalProject upperProject = (LogicalProject) rewritten; + Assertions.assertEquals(first.getExprId(), upperProject.getProjects().get(0).getExprId()); + Assertions.assertEquals(second.getExprId(), upperProject.getProjects().get(1).getExprId()); + Assertions.assertEquals(id.getExprId(), upperProject.getProjects().get(2).getExprId()); + + LogicalTopN topN = (LogicalTopN) upperProject.child(0); + LogicalProject rewrittenForwardingProject = (LogicalProject) topN.child(0); + LogicalProject rewrittenLowerProject = (LogicalProject) rewrittenForwardingProject.child(0); + for (LogicalProject project : ImmutableList.of(rewrittenForwardingProject, rewrittenLowerProject)) { + Assertions.assertEquals(2, project.getProjects().size()); + Assertions.assertEquals(id.getExprId(), project.getProjects().get(0).getExprId()); + Assertions.assertEquals(source.getExprId(), project.getProjects().get(1).getExprId()); + } + } + + private void assertRepeatedIdentityAliasesPulledUp(int aliasCount) { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + Slot id = scan1.getOutput().get(0); + Slot source = scan1.getOutput().get(1); + ImmutableList.Builder aliasesBuilder = ImmutableList.builder(); + for (int i = 0; i < aliasCount; i++) { + aliasesBuilder.add(source.alias("alias_" + i)); + } + List aliases = aliasesBuilder.build(); + ImmutableList.Builder projectsBuilder = ImmutableList.builder(); + projectsBuilder.addAll(aliases).add(id); + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .projectExprs(projectsBuilder.build()) + .topN(3, 0, ImmutableList.of(aliasCount)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(connectContext, plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .getPlan(); + + LogicalProject upperProject = (LogicalProject) rewritten; + Assertions.assertEquals(aliasCount + 1, upperProject.getProjects().size()); + for (int i = 0; i < aliasCount; i++) { + Assertions.assertEquals(aliases.get(i).getExprId(), upperProject.getProjects().get(i).getExprId()); + Assertions.assertEquals(aliases.get(i).getName(), upperProject.getProjects().get(i).getName()); + Assertions.assertEquals(aliases.get(i).getDataType(), upperProject.getProjects().get(i).getDataType()); + Assertions.assertEquals(aliases.get(i).nullable(), upperProject.getProjects().get(i).nullable()); + } + Assertions.assertEquals(id.getExprId(), upperProject.getProjects().get(aliasCount).getExprId()); + + LogicalTopN topN = (LogicalTopN) upperProject.child(0); + Assertions.assertEquals(id.getExprId(), topN.getOrderKeys().get(0).getExpr().getInputSlots().iterator() + .next().getExprId()); + LogicalProject lowerProject = (LogicalProject) topN.child(0); + Assertions.assertEquals(2, lowerProject.getProjects().size()); + Assertions.assertEquals(id.getExprId(), lowerProject.getProjects().get(0).getExprId()); + Assertions.assertEquals(source.getExprId(), lowerProject.getProjects().get(1).getExprId()); + Assertions.assertEquals(1, lowerProject.getProjects().stream() + .filter(expression -> expression.getExprId().equals(source.getExprId())) + .count()); + } + @Test void testNotPullUpScoreExpression() { List exprs = ImmutableList.of(