Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.cache.NereidsSqlCacheManager;
import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.common.util.DynamicPartitionUtil;
import org.apache.doris.common.util.MetaLockUtils;
Expand Down Expand Up @@ -411,8 +410,7 @@ private boolean processAlterOlapTableInternal(List<AlterOp> alterOps, OlapTable
}
olapTable.writeLock();
try {
NereidsSqlCacheManager sqlCacheManager = Env.getCurrentEnv().getSqlCacheManager();
sqlCacheManager.invalidateAboutTable(olapTable);
Env.getCurrentEnv().onTableChanged(olapTable);
} finally {
olapTable.writeUnlock();
}
Expand Down Expand Up @@ -857,8 +855,11 @@ private void replaceTableInternal(Database db, OlapTable origTable, OlapTable ne
// rename origin table name to new table name and add it to database
origTable.checkAndSetName(newTblName, false);
db.registerTable(origTable);
} else {

}
// Advance revisions after the name mapping changes, retaining both original dependency identities.
Env.getCurrentEnv().onTableChanged(origTable, oldTblName);
Env.getCurrentEnv().onTableChanged(newTbl, newTblName);
if (!swapTable) {
// not swap, the origin table is not used anymore, need to drop all its tablets.
// put original table to recycle bin.
if (isForce) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -649,6 +649,7 @@ private void onFinished(OlapTable tbl) {
tbl.getIndexIdToSchema(true));
}
tbl.rebuildFullSchema();
Env.getCurrentEnv().onTableChanged(tbl);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -856,6 +856,7 @@ private void onFinished(OlapTable tbl) {
tbl.setStorageFormat(storageFormat);
}
tbl.setColumnSeqMapping(columnSeqMapping);
Env.getCurrentEnv().onTableChanged(tbl);
}

/*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2254,10 +2254,7 @@ protected Status allTabletCommitted(boolean isReplay) {
}

protected void updateOlapTablesVersion(Database db, boolean isReplay) {
if (Env.getCurrentEnv().invalidCacheForCloud()) {
return;
}

boolean cloudCacheInvalidated = Env.getCurrentEnv().invalidCacheForCloud();
for (String tableName : jobInfo.backupOlapTableObjects.keySet()) {
Table tbl = db.getTableNullable(jobInfo.getAliasByOriginNameIfSet(tableName));
if (tbl == null) {
Expand All @@ -2273,8 +2270,11 @@ protected void updateOlapTablesVersion(Database db, boolean isReplay) {
continue;
}
try {
long nextVersion = olapTbl.getNextVersion();
olapTbl.updateVisibleVersionAndTime(nextVersion, System.currentTimeMillis());
if (!cloudCacheInvalidated) {
long nextVersion = olapTbl.getNextVersion();
olapTbl.updateVisibleVersionAndTime(nextVersion, System.currentTimeMillis());
}
env.onTableChanged(olapTbl);
} finally {
olapTbl.writeUnlock();
}
Expand Down
105 changes: 93 additions & 12 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
Original file line number Diff line number Diff line change
Expand Up @@ -170,9 +170,11 @@
import org.apache.doris.mysql.privilege.AccessControllerManager;
import org.apache.doris.mysql.privilege.Auth;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.SqlCacheContext.FullTableName;
import org.apache.doris.nereids.jobs.load.LabelProcessor;
import org.apache.doris.nereids.lineage.LineageEventProcessor;
import org.apache.doris.nereids.stats.HboPlanStatisticsManager;
import org.apache.doris.nereids.stats.SimpleAggCacheMgr;
import org.apache.doris.nereids.trees.plans.commands.AdminSetFrontendConfigCommand;
import org.apache.doris.nereids.trees.plans.commands.AdminSetPartitionVersionCommand;
import org.apache.doris.nereids.trees.plans.commands.AdminSetReplicaStatusCommand;
Expand Down Expand Up @@ -3640,8 +3642,8 @@ public boolean invalidCacheForCloud() {
if (Config.isCloudMode()) {
// the cloud mode will not update table version when use recover partition,
// so we should invalidate all cache to avoid bugs
getSqlCacheManager().invalidateAll();
getSortedPartitionsCacheManager().invalidateAll();
invalidateCache("SQL", () -> getSqlCacheManager().invalidateAll());
invalidateCache("sorted partitions", () -> getSortedPartitionsCacheManager().invalidateAll());
return true;
}
return false;
Expand Down Expand Up @@ -5897,6 +5899,7 @@ public void replayRenameTable(TableInfo tableInfo) throws MetaNotFoundException
db.unregisterTable(tableName);
table.setName(newTableName);
db.registerTable(table);
onTableChanged(table, tableName);
constraintManager.renameTable(
TableNameInfoUtils.fromDb(db, tableName),
TableNameInfoUtils.fromDb(db, newTableName));
Expand Down Expand Up @@ -6500,14 +6503,14 @@ public void modifyTableProperties(Database db, OlapTable table, Map<String, Stri
properties);
editLog.logModifyTableProperties(info);
if (properties.containsKey(RowTtlConfig.TTL_PROPERTY)) {
getSqlCacheManager().invalidateAboutTable(table);
onTableChanged(table);
}
}

public void updateBinlogConfig(Database db, OlapTable table, BinlogConfig newBinlogConfig) {
Preconditions.checkArgument(table.isWriteLockHeldByCurrentThread());
table.setBinlogConfig(newBinlogConfig);
getSqlCacheManager().invalidateAboutTable(table);
onTableChanged(table);
ModifyTablePropertyOperationLog info =
new ModifyTablePropertyOperationLog(db.getId(), table.getId(), table.getName(),
newBinlogConfig.toProperties());
Expand All @@ -6519,7 +6522,7 @@ public void updateBinlogConfig(Database db, OlapTable table, BinlogConfig newBin
public void updateChangelogConfig(Database db, OlapTable table, ChangelogConfig newChangelogConfig) {
Preconditions.checkArgument(table.isWriteLockHeldByCurrentThread());
table.setChangelogConfig(newChangelogConfig);
getSqlCacheManager().invalidateAboutTable(table);
onTableChanged(table);
ModifyTablePropertyOperationLog info =
new ModifyTablePropertyOperationLog(db.getId(), table.getId(), table.getName(),
newChangelogConfig.toProperties());
Expand Down Expand Up @@ -6555,9 +6558,6 @@ public void replayModifyTableProperty(short opCode, ModifyTablePropertyOperation
// need to replay partition info meta
switch (opCode) {
case OperationType.OP_MODIFY_TABLE_PROPERTIES:
if (properties.containsKey(RowTtlConfig.TTL_PROPERTY)) {
getSqlCacheManager().invalidateAboutTable(olapTable);
}
for (Partition partition : olapTable.getPartitions()) {
olapTable.getPartitionInfo().setIsInMemory(partition.getId(), tableProperty.isInMemory());
// storage policy re-use modify in memory
Expand All @@ -6570,11 +6570,11 @@ public void replayModifyTableProperty(short opCode, ModifyTablePropertyOperation
break;
case OperationType.OP_UPDATE_BINLOG_CONFIG:
// buildProperty has rebuilt the binlog and the changelog config from the table properties.
getSqlCacheManager().invalidateAboutTable(olapTable);
break;
default:
break;
}
onTableChanged(olapTable);
} finally {
olapTable.writeUnlock();
}
Expand Down Expand Up @@ -6940,6 +6940,7 @@ public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundExcep
ExternalCatalog ctl = (ExternalCatalog) catalogMgr.getCatalog(info.getCtl());
if (ctl != null) {
ctl.replayTruncateTable(info);
onTableChanged(info.getCtl(), info.getDb(), info.getTable());
}
}
}
Expand Down Expand Up @@ -7332,7 +7333,7 @@ public void setTableStatusInternal(String dbName, String tableName, OlapTableSta
LOG.warn("ignore set same state {} for table {}. is replay: {}.",
olapTable.getState(), tableName, isReplay);
}
Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(olapTable);
onTableChanged(olapTable);
} finally {
olapTable.writeUnlock();
}
Expand Down Expand Up @@ -7440,7 +7441,7 @@ private void setReplicaVersionInternal(long tabletId, long backendId, Long versi
LOG.info("set replica {} of tablet {} on backend {} as version {}, last success version {}, "
+ "last failed version {}, update time {}. is replay: {}", replica.getId(), tabletId,
backendId, version, lastSuccessVersion, lastFailedVersion, updateTime, isReplay);
Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(table);
onTableChanged(table);
} finally {
table.writeUnlock();
}
Expand Down Expand Up @@ -7521,7 +7522,7 @@ public int setPartitionVersionInternal(String database, String table, long parti
+ " {}.", partitionId, oldVersion, visibleVersion, database, table, isReplay);
}

Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(olapTable);
onTableChanged(olapTable);
} finally {
olapTable.writeUnlock();
}
Expand Down Expand Up @@ -7820,6 +7821,86 @@ public NereidsSortedPartitionsCacheManager getSortedPartitionsCacheManager() {
return sortedPartitionsCacheManager;
}

/** Invalidate derived query caches after applying an existing table metadata journal. */
public void onTableChanged(TableIf table) {
onTableChanged(table, table.getName());
}

/** Notify after mutation while retaining the original name of a renamed or replaced table. */
public void onTableChanged(TableIf table, String originalName) {
DatabaseIf<?> db = table.getDatabase();
FullTableName name = db == null || db.getCatalog() == null ? null
: new FullTableName(db.getCatalog().getName(), db.getFullName(), originalName);
onTableChanged(table.getId(), name, table instanceof OlapTable);
}

/** The table may have been removed by the operation whose journal is being replayed. */
public void onTableChanged(long dbId, long tableId) {
Database db = getInternalCatalog().getDbNullable(dbId);
Table table = db == null ? null : db.getTableNullable(tableId);
if (table != null) {
onTableChanged(table);
} else {
onTableChanged(tableId, null, true);
}
}

private void onTableChanged(long tableId, FullTableName name, boolean olapTable) {
invalidateCache("SQL", () -> getSqlCacheManager().invalidateAboutTable(tableId, name));
invalidateCache("sorted partitions", () -> {
getSortedPartitionsCacheManager().invalidateTable(tableId);
if (name != null) {
getSortedPartitionsCacheManager().invalidateTable(name.catalog, name.db, name.table);
}
});
if (name != null) {
invalidateCache("MTMV rewrite", () -> getMtmvService().getRelationManager()
.invalidateRewriteCaches(name.catalog, name.db, name.table));
}
if (olapTable) {
invalidateCache("simple aggregates", () -> SimpleAggCacheMgr.internalInstance().invalidateTable(tableId));
}
}

/** Name-based external refresh logs can arrive before a local table object has been loaded. */
public void onTableChanged(String catalog, String db, String table) {
if (Strings.isNullOrEmpty(catalog) || InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalog)) {
Database database = getInternalCatalog().getDbNullable(db);
Table current = database == null ? null : database.getTableNullable(table);
if (current != null) {
onTableChanged(current);
return;
}
catalog = InternalCatalog.INTERNAL_CATALOG_NAME;
}
onTableChanged(-1L, new FullTableName(catalog, db, table), false);
}

public void onDatabaseChanged(String catalog, String db) {
invalidateCache("SQL", () -> getSqlCacheManager().invalidateAboutDatabase(catalog, db));
invalidateCache("sorted partitions", () -> getSortedPartitionsCacheManager().invalidateDatabase(catalog, db));
invalidateCache("MTMV rewrite", () -> getMtmvService().getRelationManager()
.invalidateRewriteCaches(catalog, db, null));
}

public void onCatalogChanged(String catalog) {
invalidateCache("SQL", () -> getSqlCacheManager().invalidateAboutCatalog(catalog));
invalidateCache("sorted partitions", () -> getSortedPartitionsCacheManager().invalidateCatalog(catalog));
invalidateCache("MTMV rewrite", () -> getMtmvService().getRelationManager()
.invalidateRewriteCaches(catalog, null, null));
}

private void invalidateCache(String cache, Runnable invalidation) {
if (isCheckpointThread()) {
return;
}
try {
invalidation.run();
} catch (Throwable t) {
LOG.warn("Failed to invalidate {} cache after applying a metadata change", cache, t);
}
}

public SplitSourceManager getSplitSourceManager() {
return splitSourceManager;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,19 @@ public void invalidateTable(String catalog, String db, String table) {
partitionCaches.invalidate(key);
}

/** Match the stable ID as well, including entries under a table's previous name. */
public void invalidateTable(long tableId) {
partitionCaches.asMap().entrySet().removeIf(entry -> entry.getValue().tableId == tableId);
}

public void invalidateDatabase(String catalog, String db) {
partitionCaches.asMap().keySet().removeIf(key -> key.catalog.equals(catalog) && key.db.equals(db));
}

public void invalidateCatalog(String catalog) {
partitionCaches.asMap().keySet().removeIf(key -> key.catalog.equals(catalog));
}

public Optional<SortedPartitionRanges<?>> get(
SupportBinarySearchFilteringPartitions table, CatalogRelation scan) {
ConnectContext connectContext = ConnectContext.get();
Expand Down
Loading
Loading