Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 38 additions & 25 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,7 @@ private MTMVStatus applyStatus(MTMVStatus newStatus) {
// only can update state, refresh state will be change by add task
this.schemaChangeVersion++;
this.refreshSnapshot = new MTMVRefreshSnapshot();
invalidateRewriteCache();
return this.status.updateStateAndDetail(newStatus);
}

Expand Down Expand Up @@ -426,7 +427,12 @@ public boolean addTaskResult(AlterMTMV alterMTMV, boolean isReplay) {
applyRefreshedEpochs(capturedEpochs);
}
}
boolean publishCache = false;
if (task.getStatus() == TaskStatus.SUCCESS) {
publishCache = needUpdateCache && cacheGeneration == rewriteCacheGeneration && !isDropped;
// A successful refresh can restore a changed definition, including after several
// partial refreshes. Discard caches and builds from the previous snapshot coverage.
invalidateRewriteCache();
this.status.setState(MTMVState.NORMAL);
this.status.setSchemaChangeDetail(null);
this.status.setRefreshState(MTMVRefreshState.SUCCESS);
Expand All @@ -437,21 +443,6 @@ public boolean addTaskResult(AlterMTMV alterMTMV, boolean isReplay) {
ivmInfo.setPlanSignature(refreshedIvmPlanSignature);
}
}
// The refresh publishes a new plan, so every cache built before this commit is stale.
// Bump before publishing so an in-flight build cannot pass its generation check later.
boolean publishCache = needUpdateCache && cacheGeneration == rewriteCacheGeneration && !isDropped;
rewriteCacheGeneration++;
if (needUpdateCache) {
MTMVCacheManager manager = Env.getCurrentEnv().getMtmvCacheManager();
if (publishCache && mtmvCacheWithGuard != null) {
manager.put(this.id, true, mtmvCacheWithGuard);
} else {
manager.invalidate(this.id);
}
if (publishCache && mtmvCacheWithoutGuard != null) {
manager.put(this.id, false, mtmvCacheWithoutGuard);
}
}
} else {
this.status.setRefreshState(MTMVRefreshState.FAIL);
}
Expand All @@ -465,6 +456,17 @@ public boolean addTaskResult(AlterMTMV alterMTMV, boolean isReplay) {
snapshotsToWrite = snapshotsOfCleanPartitions(partitionSnapshots);
}
this.refreshSnapshot.updateSnapshots(snapshotsToWrite, getPartitionNames());
if (publishCache) {
// Publish only after the refreshed partitions have been recorded. A partial
// refresh may restore NORMAL while other partitions still use the old definition.
MTMVCacheManager manager = Env.getCurrentEnv().getMtmvCacheManager();
if (mtmvCacheWithGuard != null) {
manager.put(this.id, true, cacheWithValidOutputPredicates(mtmvCacheWithGuard));
}
if (mtmvCacheWithoutGuard != null) {
manager.put(this.id, false, cacheWithValidOutputPredicates(mtmvCacheWithoutGuard));
}
}
Env.getCurrentEnv().getMtmvService()
.refreshComplete(this, relation, task);
if (isReplay) {
Expand Down Expand Up @@ -726,11 +728,11 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws
MTMVCache cached;
readMvLock();
try {
cacheGeneration = rewriteCacheGeneration;
cached = manager.isEnabled() ? manager.getIfPresent(this.id, guarded) : null;
if (cached == null && statementContext != null) {
cached = statementContext.getQueryLocalMtmvCache(this.id, guarded);
cached = statementContext.getQueryLocalMtmvCache(this.id, guarded, cacheGeneration);
}
cacheGeneration = rewriteCacheGeneration;
} finally {
readMvUnlock();
}
Expand All @@ -744,23 +746,24 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws
// Someone invalidated between our snapshot and now; drop the stale build and retry.
continue;
}
MTMVCache published = cacheWithValidOutputPredicates(generated);
if (manager.isEnabled()) {
MTMVCache existing = manager.getIfPresent(this.id, guarded);
if (existing != null) {
return existing;
}
if (!isDropped) {
manager.put(this.id, guarded, generated);
manager.put(this.id, guarded, published);
}
} else if (statementContext != null && !isDropped) {
// Global cache is disabled (maximumSize=0); keep one copy for this statement only.
MTMVCache existing = statementContext.getQueryLocalMtmvCache(this.id, guarded);
MTMVCache existing = statementContext.getQueryLocalMtmvCache(this.id, guarded, cacheGeneration);
if (existing != null) {
return existing;
}
statementContext.putQueryLocalMtmvCache(this.id, guarded, generated);
statementContext.putQueryLocalMtmvCache(this.id, guarded, cacheGeneration, published);
}
return generated;
return published;
} finally {
readMvUnlock();
}
Expand Down Expand Up @@ -1769,6 +1772,16 @@ public void writeMvUnlock() {
this.mvRwLock.writeLock().unlock();
}

private MTMVCache cacheWithValidOutputPredicates(MTMVCache cache) {
// A cache built from the current definition can prove facts about stored rows only after
// every partition has been refreshed. Callers hold the MV lock while publishing it.
if (status.getState() == MTMVState.SCHEMA_CHANGE
Comment thread
foxtail463 marked this conversation as resolved.
Resolved
|| !refreshSnapshot.getPartitionSnapshots().keySet().containsAll(getPartitionNames())) {
return cache.withoutOutputPredicates();
}
return cache;
}

// toString() is not easy to find where to call the method
public String toInfoString() {
final StringBuilder sb = new StringBuilder("MTMV{");
Expand Down Expand Up @@ -1806,8 +1819,8 @@ public void compatible(CatalogMgr catalogMgr) {
Env.getCurrentEnv().getMtmvService().registerMTMV(this, this.getDatabase().getId());
} catch (Throwable e) {
LOG.warn("MTMV compatible failed, dbName: {}, mvName: {}, errMsg: {}", getDBName(), name, e.getMessage());
status.setState(MTMVState.SCHEMA_CHANGE);
status.setSchemaChangeDetail("compatible failed, please refresh or recreate it, reason: " + e.getMessage());
alterStatus(new MTMVStatus(MTMVState.SCHEMA_CHANGE,
"compatible failed, please refresh or recreate it, reason: " + e.getMessage()));
}
}

Expand Down Expand Up @@ -1840,8 +1853,8 @@ public void gsonPostProcess() throws IOException {
}
if (refreshInfo != null && refreshInfo.getRefreshMethod() == null) {
LOG.warn("MTMV {} has unknown refresh method, marking as schema change", name);
status.setState(MTMVState.SCHEMA_CHANGE);
status.setSchemaChangeDetail("Unknown refresh method detected during deserialization");
alterStatus(new MTMVStatus(MTMVState.SCHEMA_CHANGE,
"Unknown refresh method detected during deserialization"));
}
Map<String, MTMVRefreshPartitionSnapshot> partitionSnapshots = refreshSnapshot.getPartitionSnapshots();
compatiblePctSnapshot(partitionSnapshots);
Expand Down
36 changes: 34 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.rules.rewrite.EliminateSort;
import org.apache.doris.nereids.rules.rewrite.MergeProjectable;
import org.apache.doris.nereids.rules.rewrite.PullUpPredicates;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
Expand All @@ -44,12 +46,14 @@
import org.apache.doris.statistics.model.Statistics;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;

/**
* The cache for materialized view cache
Expand All @@ -65,13 +69,17 @@ public class MTMVCache {
private final Plan originalFinalPlan;
private final Statistics statistics;
private final List<Pair<Plan, StructInfo>> partRulesRewrittenPlanAndStructInfos;
// Guarantees expressed only in the definition's output slots, including session-variable guards.
private final Set<Expression> outputPredicates;

public MTMVCache(Pair<Plan, StructInfo> allRulesRewrittenPlanAndStructInfo, Plan originalFinalPlan,
Statistics statistics, List<Pair<Plan, StructInfo>> partRulesRewrittenPlanAndStructInfos) {
Statistics statistics, List<Pair<Plan, StructInfo>> partRulesRewrittenPlanAndStructInfos,
Set<Expression> outputPredicates) {
this.allRulesRewrittenPlanAndStructInfo = allRulesRewrittenPlanAndStructInfo;
this.originalFinalPlan = originalFinalPlan;
this.statistics = statistics;
this.partRulesRewrittenPlanAndStructInfos = partRulesRewrittenPlanAndStructInfos;
this.outputPredicates = ImmutableSet.copyOf(outputPredicates);
}

public Pair<Plan, StructInfo> getAllRulesRewrittenPlanAndStructInfo() {
Expand All @@ -82,6 +90,16 @@ public Plan getOriginalFinalPlan() {
return originalFinalPlan;
}

public Set<Expression> getOutputPredicates() {
return outputPredicates;
}

/** Keep the definition plan without claiming that its predicates hold for stored rows. */
public MTMVCache withoutOutputPredicates() {
return new MTMVCache(allRulesRewrittenPlanAndStructInfo, originalFinalPlan,
statistics, partRulesRewrittenPlanAndStructInfos, ImmutableSet.of());
}

public Statistics getStatistics() {
return statistics;
}
Expand Down Expand Up @@ -142,6 +160,19 @@ public static MTMVCache from(String defSql,
.orElse(rewritePlan);
Pair<Plan, StructInfo> finalPlanStructInfoPair = constructPlanAndStructInfo(
addGuardRewritePlan, cascadesContext);
// Analysis records nondeterministic functions before constant folding erases their origin.
// Replanning current_date(), for example, cannot establish a guarantee about rows stored
// by an earlier refresh. Do not export output guarantees from such a definition.
// Alias UDF definitions can also change between refresh and cache reconstruction,
// even when the expanded function body is deterministic.
// A lower MV may have refreshed since this MV's rows were stored. Derive guarantees
// from this definition alone, without importing the lower MV's current output facts.
Set<Expression> outputPredicates = (cascadesContext.getStatementContext().hasNondeterministic()
Comment thread
foxtail463 marked this conversation as resolved.
Resolved
Comment thread
foxtail463 marked this conversation as resolved.
Resolved
|| cascadesContext.getStatementContext().hasAliasUdf())
Comment thread
foxtail463 marked this conversation as resolved.
Resolved
? ImmutableSet.of()
: finalPlanStructInfoPair.key().accept(new PullUpPredicates(true, cascadesContext), null).stream()
.filter(predicate -> !predicate.containsVolatileExpression())
.collect(ImmutableSet.toImmutableSet());
List<Pair<Plan, StructInfo>> tmpPlanUsedForRewrite = new ArrayList<>();
for (Plan plan : cascadesContext.getStatementContext().getTmpPlanForMvRewrite()) {
Plan addGuardplan = exprRewriter
Expand All @@ -150,7 +181,8 @@ public static MTMVCache from(String defSql,
tmpPlanUsedForRewrite.add(constructPlanAndStructInfo(addGuardplan, cascadesContext));
}
return new MTMVCache(finalPlanStructInfoPair, addGuardRewritePlan, needCost
? cascadesContext.getMemo().getRoot().getStatistics() : null, tmpPlanUsedForRewrite);
? cascadesContext.getMemo().getRoot().getStatistics() : null, tmpPlanUsedForRewrite,
outputPredicates);
} finally {
createCacheContext.getStatementContext().setForceRecordTmpPlan(false);
mvSqlStatementContext.setForceRecordTmpPlan(false);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,9 @@ public enum TableFrom {

private boolean hasNondeterministic = false;

// Alias UDF calls disappear after expansion; retain their definition dependency.
private boolean hasAliasUdf = false;

// hasUnknownColStats true if any column stats in the tables used by this sql is
// unknown
// the algorithm to derive plan when column stats are unknown is implemented in
Expand Down Expand Up @@ -339,8 +342,9 @@ public enum TableFrom {
private final Map<BaseTableInfo, Collection<Partition>> mvCanRewritePartitionsMap = new HashMap<>();

// When the Env-wide MTMVCacheManager is disabled (mtmv_cache_manage_num=0), reuse rewrite plans
// in the same statement so multiple rewrite paths do not rebuild the same MV plan.
private final Map<Pair<Long, Boolean>, MTMVCache> queryLocalMtmvCaches = new HashMap<>();
// in the same statement so multiple rewrite paths do not rebuild the same MV plan. Each value
// carries the MV's cache generation because a planning retry can reuse this statement context.
private final Map<Pair<Long, Boolean>, Pair<Long, MTMVCache>> queryLocalMtmvCaches = new HashMap<>();

/// for dictionary sink.
private List<Backend> usedBackendsDistributing; // report used backends after done distribute planning.
Expand Down Expand Up @@ -628,6 +632,14 @@ public boolean hasNondeterministic() {
return hasNondeterministic;
}

public void recordAliasUdf() {
hasAliasUdf = true;
}

public boolean hasAliasUdf() {
return hasAliasUdf;
}

public ConnectContext getConnectContext() {
return connectContext;
}
Expand Down Expand Up @@ -1677,12 +1689,13 @@ public void addMaterializationRewrittenSuccess(List<String> materializationQuali
this.materializationRewrittenSuccessSet.add(materializationQualifier);
}

public MTMVCache getQueryLocalMtmvCache(long mtmvId, boolean guarded) {
return queryLocalMtmvCaches.get(Pair.of(mtmvId, guarded));
public MTMVCache getQueryLocalMtmvCache(long mtmvId, boolean guarded, long generation) {
Pair<Long, MTMVCache> entry = queryLocalMtmvCaches.get(Pair.of(mtmvId, guarded));
return entry != null && entry.first == generation ? entry.second : null;
}

public void putQueryLocalMtmvCache(long mtmvId, boolean guarded, MTMVCache cache) {
queryLocalMtmvCaches.put(Pair.of(mtmvId, guarded), cache);
public void putQueryLocalMtmvCache(long mtmvId, boolean guarded, long generation, MTMVCache cache) {
queryLocalMtmvCaches.put(Pair.of(mtmvId, guarded), Pair.of(generation, cache));
}

public Multimap<List<String>, Pair<RelationId, Set<String>>> getTableUsedPartitionNameMap() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import org.apache.doris.mysql.MysqlCommand;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.SqlCacheContext;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.analyzer.Scope;
import org.apache.doris.nereids.analyzer.UnboundAlias;
import org.apache.doris.nereids.analyzer.UnboundFunction;
Expand Down Expand Up @@ -199,7 +198,12 @@ public Expression analyze(Expression expression) {

/** analyze */
public Expression analyze(Expression expression, ExpressionRewriteContext context) {
return expression.accept(this, context);
Expression analyzed = expression.accept(this, context);
// Inspect the bound tree to include parser-bound functions and expanded alias UDF bodies.
if (context != null && analyzed.containsNondeterministic()) {
context.cascadesContext.getStatementContext().setHasNondeterministic(true);
}
return analyzed;
}

@Override
Expand Down Expand Up @@ -620,14 +624,13 @@ public Expression visitUnboundFunction(UnboundFunction unboundFunction, Expressi
});
}

if (builder instanceof AliasUdfBuilder && context != null) {
context.cascadesContext.getStatementContext().recordAliasUdf();
}
Pair<? extends Expression, ? extends BoundFunction> buildResult = builder.build(functionName, arguments);
buildResult.second.checkOrderExprIsValid();
Optional<SqlCacheContext> sqlCacheContext = Optional.empty();

if (!buildResult.second.isDeterministic() && context != null) {
StatementContext statementContext = context.cascadesContext.getStatementContext();
statementContext.setHasNondeterministic(true);
}
if (wantToParseSqlFromSqlCache) {
sqlCacheContext = context.cascadesContext.getStatementContext().getSqlCacheContext();
if (builder instanceof AliasUdfBuilder
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,10 @@ public MaterializationContext(Plan plan, Plan originalPlan,
this.enableRecordFailureDetail = parsedStatement != null && parsedStatement.isExplain()
&& ExplainLevel.MEMO_PLAN == parsedStatement.getExplainOptions().getExplainLevel();
// Construct materialization struct info, catch exception which may cause planner roll back
this.structInfo = structInfo == null
? constructStructInfo(plan, originalPlan, cascadesContext).orElseGet(() -> null)
: structInfo;
this.structInfo = (structInfo == null
? constructStructInfo(plan, originalPlan, cascadesContext)
: Optional.of(structInfo))
.map(info -> info.withoutRedundantMvFilters(cascadesContext)).orElse(null);
this.available = this.structInfo != null;
if (available) {
this.planOutputShuttledExpressions = this.structInfo.getPlanOutputShuttledExpressions();
Expand Down
Loading
Loading