Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.cache.NereidsSqlCacheManager;
import org.apache.doris.common.util.DynamicPartitionUtil;
import org.apache.doris.common.util.MetaLockUtils;
import org.apache.doris.common.util.PropertyAnalyzer;
Expand Down Expand Up @@ -364,8 +363,7 @@ private boolean processAlterOlapTableInternal(List<AlterClause> alterClauses, Ol

olapTable.writeLock();
try {
NereidsSqlCacheManager sqlCacheManager = Env.getCurrentEnv().getSqlCacheManager();
sqlCacheManager.invalidateAboutTable(olapTable);
Env.getCurrentEnv().onTableChanged(olapTable);
} finally {
olapTable.writeUnlock();
}
Expand Down Expand Up @@ -840,8 +838,11 @@ private void replaceTableInternal(Database db, OlapTable origTable, OlapTable ne
// rename origin table name to new table name and add it to database
origTable.checkAndSetName(newTblName, false);
db.registerTable(origTable);
} else {

}
// Advance revisions after the name mapping changes, retaining both original dependency identities.
Env.getCurrentEnv().onTableChanged(origTable, oldTblName);
Env.getCurrentEnv().onTableChanged(newTbl, newTblName);
if (!swapTable) {
// not swap, the origin table is not used anymore, need to drop all its tablets.
// put original table to recycle bin.
if (isForce) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,7 @@ private void onFinished(OlapTable tbl) {
tbl.getIndexIdToSchema(true));
}
tbl.rebuildFullSchema();
Env.getCurrentEnv().onTableChanged(tbl);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -839,6 +839,7 @@ private void onFinished(OlapTable tbl) {
if (storageFormat == TStorageFormat.V2) {
tbl.setStorageFormat(storageFormat);
}
Env.getCurrentEnv().onTableChanged(tbl);
}

/*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2233,10 +2233,7 @@ protected Status allTabletCommitted(boolean isReplay) {
}

protected void updateOlapTablesVersion(Database db, boolean isReplay) {
if (Env.getCurrentEnv().invalidCacheForCloud()) {
return;
}

boolean cloudCacheInvalidated = Env.getCurrentEnv().invalidCacheForCloud();
for (String tableName : jobInfo.backupOlapTableObjects.keySet()) {
Table tbl = db.getTableNullable(jobInfo.getAliasByOriginNameIfSet(tableName));
if (tbl == null) {
Expand All @@ -2252,8 +2249,11 @@ protected void updateOlapTablesVersion(Database db, boolean isReplay) {
continue;
}
try {
long nextVersion = olapTbl.getNextVersion();
olapTbl.updateVisibleVersionAndTime(nextVersion, System.currentTimeMillis());
if (!cloudCacheInvalidated) {
long nextVersion = olapTbl.getNextVersion();
olapTbl.updateVisibleVersionAndTime(nextVersion, System.currentTimeMillis());
}
env.onTableChanged(olapTbl);
} finally {
olapTbl.writeUnlock();
}
Expand Down
96 changes: 91 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
Original file line number Diff line number Diff line change
Expand Up @@ -171,9 +171,11 @@
import org.apache.doris.mysql.privilege.AccessControllerManager;
import org.apache.doris.mysql.privilege.Auth;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.SqlCacheContext.FullTableName;
import org.apache.doris.nereids.jobs.load.LabelProcessor;
import org.apache.doris.nereids.lineage.LineageEventProcessor;
import org.apache.doris.nereids.stats.HboPlanStatisticsManager;
import org.apache.doris.nereids.stats.SimpleAggCacheMgr;
import org.apache.doris.nereids.trees.plans.commands.AdminSetFrontendConfigCommand;
import org.apache.doris.nereids.trees.plans.commands.AdminSetPartitionVersionCommand;
import org.apache.doris.nereids.trees.plans.commands.AdminSetReplicaStatusCommand;
Expand Down Expand Up @@ -3436,8 +3438,8 @@ public boolean invalidCacheForCloud() {
if (Config.isCloudMode()) {
// the cloud mode will not update table version when use recover partition,
// so we should invalidate all cache to avoid bugs
getSqlCacheManager().invalidateAll();
getSortedPartitionsCacheManager().invalidateAll();
invalidateCache("SQL", () -> getSqlCacheManager().invalidateAll());
invalidateCache("sorted partitions", () -> getSortedPartitionsCacheManager().invalidateAll());
return true;
}
return false;
Expand Down Expand Up @@ -5735,6 +5737,7 @@ public void replayRenameTable(TableInfo tableInfo) throws MetaNotFoundException
db.unregisterTable(tableName);
table.setName(newTableName);
db.registerTable(table);
onTableChanged(table, tableName);
LOG.info("replay rename table[{}] to {}", tableName, newTableName);
} finally {
table.writeUnlock();
Expand Down Expand Up @@ -6336,6 +6339,7 @@ public void updateBinlogConfig(Database db, OlapTable table, BinlogConfig newBin
Preconditions.checkArgument(table.isWriteLockHeldByCurrentThread());

table.setBinlogConfig(newBinlogConfig);
onTableChanged(table);

ModifyTablePropertyOperationLog info =
new ModifyTablePropertyOperationLog(db.getId(), table.getId(), table.getName(),
Expand Down Expand Up @@ -6390,6 +6394,7 @@ public void replayModifyTableProperty(short opCode, ModifyTablePropertyOperation
default:
break;
}
onTableChanged(olapTable);
} finally {
olapTable.writeUnlock();
}
Expand Down Expand Up @@ -6740,6 +6745,7 @@ public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundExcep
ExternalCatalog ctl = (ExternalCatalog) catalogMgr.getCatalog(info.getCtl());
if (ctl != null) {
ctl.replayTruncateTable(info);
onTableChanged(info.getCtl(), info.getDb(), info.getTable());
}
}
}
Expand Down Expand Up @@ -7113,7 +7119,7 @@ public void setTableStatusInternal(String dbName, String tableName, OlapTableSta
LOG.warn("ignore set same state {} for table {}. is replay: {}.",
olapTable.getState(), tableName, isReplay);
}
Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(olapTable);
onTableChanged(olapTable);
} finally {
olapTable.writeUnlock();
}
Expand Down Expand Up @@ -7221,7 +7227,7 @@ private void setReplicaVersionInternal(long tabletId, long backendId, Long versi
LOG.info("set replica {} of tablet {} on backend {} as version {}, last success version {}, "
+ "last failed version {}, update time {}. is replay: {}", replica.getId(), tabletId,
backendId, version, lastSuccessVersion, lastFailedVersion, updateTime, isReplay);
Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(table);
onTableChanged(table);
} finally {
table.writeUnlock();
}
Expand Down Expand Up @@ -7302,7 +7308,7 @@ public int setPartitionVersionInternal(String database, String table, long parti
+ " {}.", partitionId, oldVersion, visibleVersion, database, table, isReplay);
}

Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(olapTable);
onTableChanged(olapTable);
} finally {
olapTable.writeUnlock();
}
Expand Down Expand Up @@ -7609,6 +7615,86 @@ public NereidsSortedPartitionsCacheManager getSortedPartitionsCacheManager() {
return sortedPartitionsCacheManager;
}

/** Invalidate derived query caches after applying an existing table metadata journal. */
public void onTableChanged(TableIf table) {
onTableChanged(table, table.getName());
}

/** Notify after mutation while retaining the original name of a renamed or replaced table. */
public void onTableChanged(TableIf table, String originalName) {
DatabaseIf<?> db = table.getDatabase();
FullTableName name = db == null || db.getCatalog() == null ? null
: new FullTableName(db.getCatalog().getName(), db.getFullName(), originalName);
onTableChanged(table.getId(), name, table instanceof OlapTable);
}

/** The table may have been removed by the operation whose journal is being replayed. */
public void onTableChanged(long dbId, long tableId) {
Database db = getInternalCatalog().getDbNullable(dbId);
Table table = db == null ? null : db.getTableNullable(tableId);
if (table != null) {
onTableChanged(table);
} else {
onTableChanged(tableId, null, true);
}
}

private void onTableChanged(long tableId, FullTableName name, boolean olapTable) {
invalidateCache("SQL", () -> getSqlCacheManager().invalidateAboutTable(tableId, name));
invalidateCache("sorted partitions", () -> {
getSortedPartitionsCacheManager().invalidateTable(tableId);
if (name != null) {
getSortedPartitionsCacheManager().invalidateTable(name.catalog, name.db, name.table);
}
});
if (name != null) {
invalidateCache("MTMV rewrite", () -> getMtmvService().getRelationManager()
.invalidateRewriteCaches(name.catalog, name.db, name.table));
}
if (olapTable) {
invalidateCache("simple aggregates", () -> SimpleAggCacheMgr.internalInstance().invalidateTable(tableId));
}
}

/** Name-based external refresh logs can arrive before a local table object has been loaded. */
public void onTableChanged(String catalog, String db, String table) {
if (Strings.isNullOrEmpty(catalog) || InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalog)) {
Database database = getInternalCatalog().getDbNullable(db);
Table current = database == null ? null : database.getTableNullable(table);
if (current != null) {
onTableChanged(current);
return;
}
catalog = InternalCatalog.INTERNAL_CATALOG_NAME;
}
onTableChanged(-1L, new FullTableName(catalog, db, table), false);
}

public void onDatabaseChanged(String catalog, String db) {
invalidateCache("SQL", () -> getSqlCacheManager().invalidateAboutDatabase(catalog, db));
invalidateCache("sorted partitions", () -> getSortedPartitionsCacheManager().invalidateDatabase(catalog, db));
invalidateCache("MTMV rewrite", () -> getMtmvService().getRelationManager()
.invalidateRewriteCaches(catalog, db, null));
}

public void onCatalogChanged(String catalog) {
invalidateCache("SQL", () -> getSqlCacheManager().invalidateAboutCatalog(catalog));
invalidateCache("sorted partitions", () -> getSortedPartitionsCacheManager().invalidateCatalog(catalog));
invalidateCache("MTMV rewrite", () -> getMtmvService().getRelationManager()
.invalidateRewriteCaches(catalog, null, null));
}

private void invalidateCache(String cache, Runnable invalidation) {
if (isCheckpointThread()) {
return;
}
try {
invalidation.run();
} catch (Throwable t) {
LOG.warn("Failed to invalidate {} cache after applying a metadata change", cache, t);
}
}

public SplitSourceManager getSplitSourceManager() {
return splitSourceManager;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@

public class MTMV extends OlapTable {
private static final Logger LOG = LogManager.getLogger(MTMV.class);
private static final int MAX_REWRITE_CACHE_BUILD_ATTEMPTS = 4;
private ReentrantReadWriteLock mvRwLock;

@SerializedName("ri")
Expand Down Expand Up @@ -489,7 +490,7 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws
boolean sessionVarsMatch = SessionVarGuardRewriter.checkSessionVariablesMatch(
currentSessionVars, this.sessionVariables);

while (true) {
for (int attempt = 0; attempt < MAX_REWRITE_CACHE_BUILD_ATTEMPTS; attempt++) {
long cacheGeneration;
readMvLock();
try {
Expand Down Expand Up @@ -524,6 +525,9 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws
writeMvUnlock();
}
}
// MV rewriting and derived traits are optional; callers already skip them on analysis failure.
throw new org.apache.doris.nereids.exceptions.AnalysisException(
"Failed to build rewrite cache for MV " + getName() + " because metadata kept changing");
}

/** Invalidate both cache variants and reject any older build still in progress. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,7 @@ default void addUniqueConstraint(String name, ImmutableList<String> columns, boo
UniqueConstraint uniqueConstraint = new UniqueConstraint(name, ImmutableSet.copyOf(columns));
checkConstraintNotExistenceUnsafe(name, uniqueConstraint, constraintMap);
constraintMap.put(name, uniqueConstraint);
notifyConstraintChanged(getConstraintRelatedTables(uniqueConstraint));
if (!replay) {
Env.getCurrentEnv().getEditLog().logAddConstraint(
new AlterConstraintLog(uniqueConstraint, this));
Expand All @@ -332,6 +333,7 @@ default void addPrimaryKeyConstraint(String name, ImmutableList<String> columns,
PrimaryKeyConstraint primaryKeyConstraint = new PrimaryKeyConstraint(name, ImmutableSet.copyOf(columns));
checkConstraintNotExistenceUnsafe(name, primaryKeyConstraint, constraintMap);
constraintMap.put(name, primaryKeyConstraint);
notifyConstraintChanged(getConstraintRelatedTables(primaryKeyConstraint));
if (!replay) {
Env.getCurrentEnv().getEditLog().logAddConstraint(
new AlterConstraintLog(primaryKeyConstraint, this));
Expand Down Expand Up @@ -363,6 +365,7 @@ default void addForeignConstraint(String name, ImmutableList<String> columns,
referencedTable);
primaryKeyConstraint.addForeignTable(this);
constraintMap.put(name, foreignKeyConstraint);
notifyConstraintChanged(getConstraintRelatedTables(foreignKeyConstraint));
if (!replay) {
Env.getCurrentEnv().getEditLog().logAddConstraint(
new AlterConstraintLog(foreignKeyConstraint, this));
Expand Down Expand Up @@ -407,6 +410,7 @@ default void dropConstraintRefWithLock(Constraint constraint) {
MetaLockUtils.writeLockTables(tables);
try {
dropConstraintRef(constraint);
notifyConstraintChanged(tables);
} finally {
MetaLockUtils.writeUnlockTables(tables);
}
Expand Down Expand Up @@ -441,6 +445,11 @@ default List<TableIf> getConstraintRelatedTables(Constraint constraint) {
return tables;
}

default void notifyConstraintChanged(List<TableIf> relatedTables) {
Env.getCurrentEnv().onTableChanged(this);
relatedTables.forEach(table -> Env.getCurrentEnv().onTableChanged(table));
}

default void dropConstraint(String name, boolean replay) {
Map<String, Constraint> constraintMap = getConstraintsMapUnsafe();
if (!constraintMap.containsKey(name)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,19 @@ public void invalidateTable(String catalog, String db, String table) {
partitionCaches.invalidate(key);
}

/** Match the stable ID as well, including entries under a table's previous name. */
public void invalidateTable(long tableId) {
partitionCaches.asMap().entrySet().removeIf(entry -> entry.getValue().tableId == tableId);
}

public void invalidateDatabase(String catalog, String db) {
partitionCaches.asMap().keySet().removeIf(key -> key.catalog.equals(catalog) && key.db.equals(db));
}

public void invalidateCatalog(String catalog) {
partitionCaches.asMap().keySet().removeIf(key -> key.catalog.equals(catalog));
}

public Optional<SortedPartitionRanges<?>> get(
SupportBinarySearchFilteringPartitions table, CatalogRelation scan) {
ConnectContext connectContext = ConnectContext.get();
Expand Down
Loading
Loading