From bdf48a30d913d99d8e67a0c30fc7eab27e48f1a8 Mon Sep 17 00:00:00 2001 From: York Cao <978176728@qq.com> Date: Fri, 11 Sep 2026 18:13:21 +0800 Subject: [PATCH 1/2] [fix](topn) Generate unique TopN lazy fetch schema Issue Number: None Related PR: None Problem Summary: Repeated aliases of one source slot could remain below TopN and produce duplicate lazy fetch slots. Classify repeated identity projections per LogicalProject by source ExprId, pull their aliases above TopN, preserve the original output schema through the existing reconstruction path, and verify the resulting physical lazy-fetch schema. TopN lazy materialization now fetches repeated aliases of the same source column only once. - Test: FE logical-rule and physical-plan unit tests; FE build - Behavior changed: Yes, repeated identity aliases are reconstructed above TopN - Does this need documentation: No --- .../rewrite/PullUpProjectExprUnderTopN.java | 31 ++++- .../postprocess/TopnLazyMaterializeTest.java | 32 +++++ .../PullUpProjectExprUnderTopNTest.java | 109 ++++++++++++++++++ 3 files changed, 168 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopN.java index b225bb891e3eae..387354112d2287 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopN.java @@ -174,9 +174,10 @@ private static void collectFromNode(Plan node, PullUpInfo info, Set bloc if (node instanceof LogicalProject) { LogicalProject project = (LogicalProject) node; Set childBlockedExprIds = new HashSet<>(blockedExprIds); + Set repeatedIdentitySourceExprIds = collectRepeatedIdentitySourceExprIds(project.getProjects()); for (NamedExpression ne : project.getProjects()) { info.addPullUpExprReplace(ne); - boolean canPullUp = canPullUp(ne); + boolean canPullUp = canPullUp(ne, repeatedIdentitySourceExprIds); if (canPullUp && !blockedExprIds.contains(ne.getExprId())) { info.addPulledUpExpr(project, ne); } @@ -275,14 +276,17 @@ private static void collectFromNode(Plan node, PullUpInfo info, Set bloc /** * Check if a named expression can be pulled up above TopN. - * Eligible: Alias with non-trivial child, not blocked, no NoneMovableFunction. + * Eligible: repeated Alias(Slot), or Alias with a non-trivial movable child. */ - static boolean canPullUp(NamedExpression ne) { + static boolean canPullUp(NamedExpression ne, Set repeatedIdentitySourceExprIds) { if (!(ne instanceof Alias)) { return false; } Expression child = ((Alias) ne).child(); - if (child instanceof Slot || child instanceof Literal) { + if (child instanceof Slot) { + return repeatedIdentitySourceExprIds.contains(((Slot) child).getExprId()); + } + if (child instanceof Literal) { return false; } if (ne.anyMatch(e -> e instanceof NoneMovableFunction)) { @@ -300,6 +304,25 @@ static boolean canPullUp(NamedExpression ne) { return true; } + private static Set collectRepeatedIdentitySourceExprIds(List projects) { + Set identitySourceExprIds = new HashSet<>(); + Set repeatedIdentitySourceExprIds = new HashSet<>(); + for (NamedExpression project : projects) { + ExprId sourceExprId; + if (project instanceof Slot) { + sourceExprId = project.getExprId(); + } else if (project instanceof Alias && project.child(0) instanceof Slot) { + sourceExprId = ((Slot) project.child(0)).getExprId(); + } else { + continue; + } + if (!identitySourceExprIds.add(sourceExprId)) { + repeatedIdentitySourceExprIds.add(sourceExprId); + } + } + return repeatedIdentitySourceExprIds; + } + private static boolean shouldBlockProjectInputs( NamedExpression ne, boolean canPullUp, Set blockedExprIds) { if (blockedExprIds.contains(ne.getExprId())) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java index ee0942c30716d9..a8adec53584b4d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java @@ -142,6 +142,38 @@ public void testLazyBaseColumnIndexUsesOriginalColumnNameForAlias() throws Excep materializeNodes.get(0).getLazyBaseColumnIndices()); } + @Test + public void testDuplicateAliasesShareOneLazyFetchSlot() throws Exception { + this.createTable("create table lazy_materialize_duplicate_alias_tbl(" + + "sort_col int, lazy_col int) " + + "duplicate key(sort_col) distributed by hash(sort_col) buckets 1 " + + "properties('replication_num' = '1')"); + String sql = "select lazy_col as first_alias, lazy_col as second_alias " + + "from lazy_materialize_duplicate_alias_tbl order by sort_col limit 1"; + + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .implement(); + PhysicalPlan plan = checker.getPhysicalPlan(); + plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan); + + List> materializeNodes = plan.collectToList( + node -> node instanceof PhysicalLazyMaterialize); + Assertions.assertEquals(1, materializeNodes.size(), plan.treeString()); + PhysicalLazyMaterialize materialize = materializeNodes.get(0); + Assertions.assertEquals(1, materialize.getRelations().size()); + Assertions.assertEquals(1, materialize.getLazySlots(materialize.getRelations().get(0)).size()); + Assertions.assertEquals(ImmutableList.of(ImmutableList.of(1)), materialize.getLazyBaseColumnIndices()); + + Assertions.assertEquals(2, plan.getOutput().size()); + Assertions.assertEquals("first_alias", plan.getOutput().get(0).getName()); + Assertions.assertEquals("second_alias", plan.getOutput().get(1).getName()); + Assertions.assertNotEquals(plan.getOutput().get(0).getExprId(), plan.getOutput().get(1).getExprId()); + Assertions.assertEquals(plan.getOutput().get(0).getDataType(), plan.getOutput().get(1).getDataType()); + Assertions.assertEquals(plan.getOutput().get(0).nullable(), plan.getOutput().get(1).nullable()); + } + @Test public void testLightSchemaChangeFalse() throws Exception { this.createTable("create table tm_lsc_false (k int, v int) duplicate key(k) " diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java index 692f978e9f0c11..497c064747b824 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java @@ -107,6 +107,115 @@ void testNotPullUpSimpleAlias() { .getPlan(); } + @Test + void testPullUpTwoAndThreeAliasesOfSameSlot() { + assertRepeatedIdentityAliasesPulledUp(2); + assertRepeatedIdentityAliasesPulledUp(3); + } + + @Test + void testPullUpDirectSlotAndRepeatedAliases() { + Slot id = scan1.getOutput().get(0); + Slot source = scan1.getOutput().get(1); + Alias first = source.alias("first"); + Alias second = source.alias("second"); + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .projectExprs(ImmutableList.of(source, first, second, id)) + .topN(3, 0, ImmutableList.of(3)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .getPlan(); + + LogicalProject upperProject = (LogicalProject) rewritten; + Assertions.assertEquals(4, upperProject.getProjects().size()); + Assertions.assertEquals(source.getExprId(), upperProject.getProjects().get(0).getExprId()); + Assertions.assertEquals(first.getExprId(), upperProject.getProjects().get(1).getExprId()); + Assertions.assertEquals(second.getExprId(), upperProject.getProjects().get(2).getExprId()); + Assertions.assertEquals(id.getExprId(), upperProject.getProjects().get(3).getExprId()); + + LogicalTopN topN = (LogicalTopN) upperProject.child(0); + LogicalProject lowerProject = (LogicalProject) topN.child(0); + Assertions.assertEquals(2, lowerProject.getProjects().size()); + Assertions.assertEquals(source.getExprId(), lowerProject.getProjects().get(0).getExprId()); + Assertions.assertEquals(id.getExprId(), lowerProject.getProjects().get(1).getExprId()); + } + + @Test + void testPullUpRepeatedAliasesThroughForwardingProject() { + Slot id = scan1.getOutput().get(0); + Slot source = scan1.getOutput().get(1); + Alias first = source.alias("first"); + Alias second = source.alias("second"); + LogicalProject lowerProject = new LogicalProject<>( + ImmutableList.of(first, second, id), scan1); + LogicalProject> forwardingProject = new LogicalProject<>( + ImmutableList.of(first.toSlot(), second.toSlot(), id), lowerProject); + LogicalPlan plan = new LogicalPlanBuilder(forwardingProject) + .topN(3, 0, ImmutableList.of(2)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .getPlan(); + + LogicalProject upperProject = (LogicalProject) rewritten; + Assertions.assertEquals(first.getExprId(), upperProject.getProjects().get(0).getExprId()); + Assertions.assertEquals(second.getExprId(), upperProject.getProjects().get(1).getExprId()); + Assertions.assertEquals(id.getExprId(), upperProject.getProjects().get(2).getExprId()); + + LogicalTopN topN = (LogicalTopN) upperProject.child(0); + LogicalProject rewrittenForwardingProject = (LogicalProject) topN.child(0); + LogicalProject rewrittenLowerProject = (LogicalProject) rewrittenForwardingProject.child(0); + for (LogicalProject project : ImmutableList.of(rewrittenForwardingProject, rewrittenLowerProject)) { + Assertions.assertEquals(2, project.getProjects().size()); + Assertions.assertEquals(id.getExprId(), project.getProjects().get(0).getExprId()); + Assertions.assertEquals(source.getExprId(), project.getProjects().get(1).getExprId()); + } + } + + private void assertRepeatedIdentityAliasesPulledUp(int aliasCount) { + Slot id = scan1.getOutput().get(0); + Slot source = scan1.getOutput().get(1); + ImmutableList.Builder aliasesBuilder = ImmutableList.builder(); + for (int i = 0; i < aliasCount; i++) { + aliasesBuilder.add(source.alias("alias_" + i)); + } + List aliases = aliasesBuilder.build(); + ImmutableList.Builder projectsBuilder = ImmutableList.builder(); + projectsBuilder.addAll(aliases).add(id); + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .projectExprs(projectsBuilder.build()) + .topN(3, 0, ImmutableList.of(aliasCount)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .getPlan(); + + LogicalProject upperProject = (LogicalProject) rewritten; + Assertions.assertEquals(aliasCount + 1, upperProject.getProjects().size()); + for (int i = 0; i < aliasCount; i++) { + Assertions.assertEquals(aliases.get(i).getExprId(), upperProject.getProjects().get(i).getExprId()); + Assertions.assertEquals(aliases.get(i).getName(), upperProject.getProjects().get(i).getName()); + Assertions.assertEquals(aliases.get(i).getDataType(), upperProject.getProjects().get(i).getDataType()); + Assertions.assertEquals(aliases.get(i).nullable(), upperProject.getProjects().get(i).nullable()); + } + Assertions.assertEquals(id.getExprId(), upperProject.getProjects().get(aliasCount).getExprId()); + + LogicalTopN topN = (LogicalTopN) upperProject.child(0); + Assertions.assertEquals(id.getExprId(), topN.getOrderKeys().get(0).getExpr().getInputSlots().iterator() + .next().getExprId()); + LogicalProject lowerProject = (LogicalProject) topN.child(0); + Assertions.assertEquals(2, lowerProject.getProjects().size()); + Assertions.assertEquals(id.getExprId(), lowerProject.getProjects().get(0).getExprId()); + Assertions.assertEquals(source.getExprId(), lowerProject.getProjects().get(1).getExprId()); + Assertions.assertEquals(1, lowerProject.getProjects().stream() + .filter(expression -> expression.getExprId().equals(source.getExprId())) + .count()); + } + @Test void testNotPullUpScoreExpression() { List exprs = ImmutableList.of( From 04118740ba597c382bdb4eb63e17638319123f7e Mon Sep 17 00:00:00 2001 From: York Cao <978176728@qq.com> Date: Tue, 15 Sep 2026 17:04:42 +0800 Subject: [PATCH 2/2] fix test --- .../nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java index 497c064747b824..996819ce67cb7c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java @@ -48,6 +48,7 @@ import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; @@ -176,6 +177,7 @@ void testPullUpRepeatedAliasesThroughForwardingProject() { } private void assertRepeatedIdentityAliasesPulledUp(int aliasCount) { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); Slot id = scan1.getOutput().get(0); Slot source = scan1.getOutput().get(1); ImmutableList.Builder aliasesBuilder = ImmutableList.builder(); @@ -190,7 +192,7 @@ private void assertRepeatedIdentityAliasesPulledUp(int aliasCount) { .topN(3, 0, ImmutableList.of(aliasCount)) .build(); - LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(connectContext, plan) .applyCustom(new PullUpProjectExprUnderTopN()) .getPlan();