Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -174,9 +174,10 @@ private static void collectFromNode(Plan node, PullUpInfo info, Set<ExprId> bloc
if (node instanceof LogicalProject) {
LogicalProject<? extends Plan> project = (LogicalProject<? extends Plan>) node;
Set<ExprId> childBlockedExprIds = new HashSet<>(blockedExprIds);
Set<ExprId> repeatedIdentitySourceExprIds = collectRepeatedIdentitySourceExprIds(project.getProjects());
for (NamedExpression ne : project.getProjects()) {
info.addPullUpExprReplace(ne);
boolean canPullUp = canPullUp(ne);
boolean canPullUp = canPullUp(ne, repeatedIdentitySourceExprIds);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Keep a blocked alias's shared source eager

For SELECT lazy_col AS first_alias, lazy_col AS second_alias FROM t ORDER BY first_alias LIMIT 1, the order key blocks only first_alias, so this per-expression decision pulls second_alias and leaves Project(lazy_col AS first_alias, lazy_col) below TopN. With topn_lazy_materialization_using_index=true, the exposed lazy_col is then classified lazy; LazySlotPruning removes it from the scan/direct projection but retains lazy_col AS first_alias, and the final Validator rejects the plan because the alias reads a slot its child no longer produces. Before this change the lower identity Alias was rejected in the index path and the TopN stayed eager, so this is a regression. Please make pull-up/materialization eligibility coherent for the whole repeated-source group (or otherwise keep the shared source eager) and cover ORDER BY alias/ordinal through final translation or result verification.

if (canPullUp && !blockedExprIds.contains(ne.getExprId())) {
info.addPulledUpExpr(project, ne);
}
Expand Down Expand Up @@ -275,14 +276,17 @@ private static void collectFromNode(Plan node, PullUpInfo info, Set<ExprId> bloc

/**
* Check if a named expression can be pulled up above TopN.
* Eligible: Alias with non-trivial child, not blocked, no NoneMovableFunction.
* Eligible: repeated Alias(Slot), or Alias with a non-trivial movable child.
*/
static boolean canPullUp(NamedExpression ne) {
static boolean canPullUp(NamedExpression ne, Set<ExprId> repeatedIdentitySourceExprIds) {
if (!(ne instanceof Alias)) {
return false;
}
Expression child = ((Alias) ne).child();
if (child instanceof Slot || child instanceof Literal) {
if (child instanceof Slot) {
return repeatedIdentitySourceExprIds.contains(((Slot) child).getExprId());
}
if (child instanceof Literal) {
return false;
}
if (ne.anyMatch(e -> e instanceof NoneMovableFunction)) {
Expand All @@ -300,6 +304,25 @@ static boolean canPullUp(NamedExpression ne) {
return true;
}

private static Set<ExprId> collectRepeatedIdentitySourceExprIds(List<NamedExpression> projects) {
Set<ExprId> identitySourceExprIds = new HashSet<>();
Set<ExprId> repeatedIdentitySourceExprIds = new HashSet<>();
for (NamedExpression project : projects) {
ExprId sourceExprId;
if (project instanceof Slot) {
sourceExprId = project.getExprId();
} else if (project instanceof Alias && project.child(0) instanceof Slot) {
sourceExprId = ((Slot) project.child(0)).getExprId();
} else {
continue;
}
if (!identitySourceExprIds.add(sourceExprId)) {
repeatedIdentitySourceExprIds.add(sourceExprId);
}
}
return repeatedIdentitySourceExprIds;
}

private static boolean shouldBlockProjectInputs(
NamedExpression ne, boolean canPullUp, Set<ExprId> blockedExprIds) {
if (blockedExprIds.contains(ne.getExprId())) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,38 @@ public void testLazyBaseColumnIndexUsesOriginalColumnNameForAlias() throws Excep
materializeNodes.get(0).getLazyBaseColumnIndices());
}

@Test
public void testDuplicateAliasesShareOneLazyFetchSlot() throws Exception {
this.createTable("create table lazy_materialize_duplicate_alias_tbl("
+ "sort_col int, lazy_col int) "
+ "duplicate key(sort_col) distributed by hash(sort_col) buckets 1 "
+ "properties('replication_num' = '1')");
String sql = "select lazy_col as first_alias, lazy_col as second_alias "
+ "from lazy_materialize_duplicate_alias_tbl order by sort_col limit 1";

PlanChecker checker = PlanChecker.from(connectContext)
.analyze(sql)
.rewrite()
.implement();
PhysicalPlan plan = checker.getPhysicalPlan();
plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan);

List<PhysicalLazyMaterialize<? extends Plan>> materializeNodes = plan.collectToList(
node -> node instanceof PhysicalLazyMaterialize);
Assertions.assertEquals(1, materializeNodes.size(), plan.treeString());
PhysicalLazyMaterialize<? extends Plan> materialize = materializeNodes.get(0);
Assertions.assertEquals(1, materialize.getRelations().size());
Assertions.assertEquals(1, materialize.getLazySlots(materialize.getRelations().get(0)).size());
Assertions.assertEquals(ImmutableList.of(ImmutableList.of(1)), materialize.getLazyBaseColumnIndices());

Assertions.assertEquals(2, plan.getOutput().size());
Assertions.assertEquals("first_alias", plan.getOutput().get(0).getName());
Assertions.assertEquals("second_alias", plan.getOutput().get(1).getName());
Assertions.assertNotEquals(plan.getOutput().get(0).getExprId(), plan.getOutput().get(1).getExprId());
Assertions.assertEquals(plan.getOutput().get(0).getDataType(), plan.getOutput().get(1).getDataType());
Assertions.assertEquals(plan.getOutput().get(0).nullable(), plan.getOutput().get(1).nullable());
}

@Test
public void testLightSchemaChangeFalse() throws Exception {
this.createTable("create table tm_lsc_false (k int, v int) duplicate key(k) "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.nereids.util.PlanConstructor;
import org.apache.doris.qe.ConnectContext;

import com.google.common.collect.ImmutableList;
import org.junit.jupiter.api.Assertions;
Expand Down Expand Up @@ -107,6 +108,116 @@ void testNotPullUpSimpleAlias() {
.getPlan();
}

@Test
Comment thread
Baymine marked this conversation as resolved.
Resolved
void testPullUpTwoAndThreeAliasesOfSameSlot() {
assertRepeatedIdentityAliasesPulledUp(2);
assertRepeatedIdentityAliasesPulledUp(3);
}

@Test
void testPullUpDirectSlotAndRepeatedAliases() {
Slot id = scan1.getOutput().get(0);
Slot source = scan1.getOutput().get(1);
Alias first = source.alias("first");
Alias second = source.alias("second");
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.projectExprs(ImmutableList.of(source, first, second, id))
.topN(3, 0, ImmutableList.of(3))
.build();

LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
.applyCustom(new PullUpProjectExprUnderTopN())
.getPlan();

LogicalProject<?> upperProject = (LogicalProject<?>) rewritten;
Assertions.assertEquals(4, upperProject.getProjects().size());
Assertions.assertEquals(source.getExprId(), upperProject.getProjects().get(0).getExprId());
Assertions.assertEquals(first.getExprId(), upperProject.getProjects().get(1).getExprId());
Assertions.assertEquals(second.getExprId(), upperProject.getProjects().get(2).getExprId());
Assertions.assertEquals(id.getExprId(), upperProject.getProjects().get(3).getExprId());

LogicalTopN<?> topN = (LogicalTopN<?>) upperProject.child(0);
LogicalProject<?> lowerProject = (LogicalProject<?>) topN.child(0);
Assertions.assertEquals(2, lowerProject.getProjects().size());
Assertions.assertEquals(source.getExprId(), lowerProject.getProjects().get(0).getExprId());
Assertions.assertEquals(id.getExprId(), lowerProject.getProjects().get(1).getExprId());
}

@Test
void testPullUpRepeatedAliasesThroughForwardingProject() {
Slot id = scan1.getOutput().get(0);
Slot source = scan1.getOutput().get(1);
Alias first = source.alias("first");
Alias second = source.alias("second");
LogicalProject<LogicalOlapScan> lowerProject = new LogicalProject<>(
ImmutableList.of(first, second, id), scan1);
LogicalProject<LogicalProject<LogicalOlapScan>> forwardingProject = new LogicalProject<>(
ImmutableList.of(first.toSlot(), second.toSlot(), id), lowerProject);
LogicalPlan plan = new LogicalPlanBuilder(forwardingProject)
.topN(3, 0, ImmutableList.of(2))
.build();

LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
.applyCustom(new PullUpProjectExprUnderTopN())
.getPlan();

LogicalProject<?> upperProject = (LogicalProject<?>) rewritten;
Assertions.assertEquals(first.getExprId(), upperProject.getProjects().get(0).getExprId());
Assertions.assertEquals(second.getExprId(), upperProject.getProjects().get(1).getExprId());
Assertions.assertEquals(id.getExprId(), upperProject.getProjects().get(2).getExprId());

LogicalTopN<?> topN = (LogicalTopN<?>) upperProject.child(0);
LogicalProject<?> rewrittenForwardingProject = (LogicalProject<?>) topN.child(0);
LogicalProject<?> rewrittenLowerProject = (LogicalProject<?>) rewrittenForwardingProject.child(0);
for (LogicalProject<?> project : ImmutableList.of(rewrittenForwardingProject, rewrittenLowerProject)) {
Assertions.assertEquals(2, project.getProjects().size());
Assertions.assertEquals(id.getExprId(), project.getProjects().get(0).getExprId());
Assertions.assertEquals(source.getExprId(), project.getProjects().get(1).getExprId());
}
}

private void assertRepeatedIdentityAliasesPulledUp(int aliasCount) {
ConnectContext connectContext = MemoTestUtils.createConnectContext();
Slot id = scan1.getOutput().get(0);
Slot source = scan1.getOutput().get(1);
ImmutableList.Builder<Alias> aliasesBuilder = ImmutableList.builder();
for (int i = 0; i < aliasCount; i++) {
aliasesBuilder.add(source.alias("alias_" + i));
}
List<Alias> aliases = aliasesBuilder.build();
ImmutableList.Builder<NamedExpression> projectsBuilder = ImmutableList.builder();
projectsBuilder.addAll(aliases).add(id);
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.projectExprs(projectsBuilder.build())
.topN(3, 0, ImmutableList.of(aliasCount))
.build();

LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(connectContext, plan)
.applyCustom(new PullUpProjectExprUnderTopN())
.getPlan();

LogicalProject<?> upperProject = (LogicalProject<?>) rewritten;
Assertions.assertEquals(aliasCount + 1, upperProject.getProjects().size());
for (int i = 0; i < aliasCount; i++) {
Assertions.assertEquals(aliases.get(i).getExprId(), upperProject.getProjects().get(i).getExprId());
Assertions.assertEquals(aliases.get(i).getName(), upperProject.getProjects().get(i).getName());
Assertions.assertEquals(aliases.get(i).getDataType(), upperProject.getProjects().get(i).getDataType());
Assertions.assertEquals(aliases.get(i).nullable(), upperProject.getProjects().get(i).nullable());
}
Assertions.assertEquals(id.getExprId(), upperProject.getProjects().get(aliasCount).getExprId());

LogicalTopN<?> topN = (LogicalTopN<?>) upperProject.child(0);
Assertions.assertEquals(id.getExprId(), topN.getOrderKeys().get(0).getExpr().getInputSlots().iterator()
.next().getExprId());
LogicalProject<?> lowerProject = (LogicalProject<?>) topN.child(0);
Assertions.assertEquals(2, lowerProject.getProjects().size());
Assertions.assertEquals(id.getExprId(), lowerProject.getProjects().get(0).getExprId());
Assertions.assertEquals(source.getExprId(), lowerProject.getProjects().get(1).getExprId());
Assertions.assertEquals(1, lowerProject.getProjects().stream()
.filter(expression -> expression.getExprId().equals(source.getExprId()))
.count());
}

@Test
void testNotPullUpScoreExpression() {
List<NamedExpression> exprs = ImmutableList.of(
Expand Down
Loading