diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 181f84a8cdba0c..6d391513ce9e42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -72,7 +72,6 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.UserException; -import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.MetaLockUtils; import org.apache.doris.common.util.PropertyAnalyzer; @@ -364,8 +363,7 @@ private boolean processAlterOlapTableInternal(List alterClauses, Ol olapTable.writeLock(); try { - NereidsSqlCacheManager sqlCacheManager = Env.getCurrentEnv().getSqlCacheManager(); - sqlCacheManager.invalidateAboutTable(olapTable); + Env.getCurrentEnv().onTableChanged(olapTable); } finally { olapTable.writeUnlock(); } @@ -840,8 +838,11 @@ private void replaceTableInternal(Database db, OlapTable origTable, OlapTable ne // rename origin table name to new table name and add it to database origTable.checkAndSetName(newTblName, false); db.registerTable(origTable); - } else { - + } + // Advance revisions after the name mapping changes, retaining both original dependency identities. + Env.getCurrentEnv().onTableChanged(origTable, oldTblName); + Env.getCurrentEnv().onTableChanged(newTbl, newTblName); + if (!swapTable) { // not swap, the origin table is not used anymore, need to drop all its tablets. // put original table to recycle bin. if (isForce) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index 5012eacab13fa3..353187b85bf9a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -647,6 +647,7 @@ private void onFinished(OlapTable tbl) { tbl.getIndexIdToSchema(true)); } tbl.rebuildFullSchema(); + Env.getCurrentEnv().onTableChanged(tbl); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 57b24fae28f624..b8306605cc5021 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -839,6 +839,7 @@ private void onFinished(OlapTable tbl) { if (storageFormat == TStorageFormat.V2) { tbl.setStorageFormat(storageFormat); } + Env.getCurrentEnv().onTableChanged(tbl); } /* diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 7c0f52b93edf59..c55ac83ee202e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -2233,10 +2233,7 @@ protected Status allTabletCommitted(boolean isReplay) { } protected void updateOlapTablesVersion(Database db, boolean isReplay) { - if (Env.getCurrentEnv().invalidCacheForCloud()) { - return; - } - + boolean cloudCacheInvalidated = Env.getCurrentEnv().invalidCacheForCloud(); for (String tableName : jobInfo.backupOlapTableObjects.keySet()) { Table tbl = db.getTableNullable(jobInfo.getAliasByOriginNameIfSet(tableName)); if (tbl == null) { @@ -2252,8 +2249,11 @@ protected void updateOlapTablesVersion(Database db, boolean isReplay) { continue; } try { - long nextVersion = olapTbl.getNextVersion(); - olapTbl.updateVisibleVersionAndTime(nextVersion, System.currentTimeMillis()); + if (!cloudCacheInvalidated) { + long nextVersion = olapTbl.getNextVersion(); + olapTbl.updateVisibleVersionAndTime(nextVersion, System.currentTimeMillis()); + } + env.onTableChanged(olapTbl); } finally { olapTbl.writeUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 2c6fab3e9a87c9..26aa27cbf4ee28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -171,9 +171,11 @@ import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.SqlCacheContext.FullTableName; import org.apache.doris.nereids.jobs.load.LabelProcessor; import org.apache.doris.nereids.lineage.LineageEventProcessor; import org.apache.doris.nereids.stats.HboPlanStatisticsManager; +import org.apache.doris.nereids.stats.SimpleAggCacheMgr; import org.apache.doris.nereids.trees.plans.commands.AdminSetFrontendConfigCommand; import org.apache.doris.nereids.trees.plans.commands.AdminSetPartitionVersionCommand; import org.apache.doris.nereids.trees.plans.commands.AdminSetReplicaStatusCommand; @@ -3436,8 +3438,8 @@ public boolean invalidCacheForCloud() { if (Config.isCloudMode()) { // the cloud mode will not update table version when use recover partition, // so we should invalidate all cache to avoid bugs - getSqlCacheManager().invalidateAll(); - getSortedPartitionsCacheManager().invalidateAll(); + invalidateCache("SQL", () -> getSqlCacheManager().invalidateAll()); + invalidateCache("sorted partitions", () -> getSortedPartitionsCacheManager().invalidateAll()); return true; } return false; @@ -5735,6 +5737,7 @@ public void replayRenameTable(TableInfo tableInfo) throws MetaNotFoundException db.unregisterTable(tableName); table.setName(newTableName); db.registerTable(table); + onTableChanged(table, tableName); LOG.info("replay rename table[{}] to {}", tableName, newTableName); } finally { table.writeUnlock(); @@ -6336,6 +6339,7 @@ public void updateBinlogConfig(Database db, OlapTable table, BinlogConfig newBin Preconditions.checkArgument(table.isWriteLockHeldByCurrentThread()); table.setBinlogConfig(newBinlogConfig); + onTableChanged(table); ModifyTablePropertyOperationLog info = new ModifyTablePropertyOperationLog(db.getId(), table.getId(), table.getName(), @@ -6390,6 +6394,7 @@ public void replayModifyTableProperty(short opCode, ModifyTablePropertyOperation default: break; } + onTableChanged(olapTable); } finally { olapTable.writeUnlock(); } @@ -6740,6 +6745,7 @@ public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundExcep ExternalCatalog ctl = (ExternalCatalog) catalogMgr.getCatalog(info.getCtl()); if (ctl != null) { ctl.replayTruncateTable(info); + onTableChanged(info.getCtl(), info.getDb(), info.getTable()); } } } @@ -7113,7 +7119,7 @@ public void setTableStatusInternal(String dbName, String tableName, OlapTableSta LOG.warn("ignore set same state {} for table {}. is replay: {}.", olapTable.getState(), tableName, isReplay); } - Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(olapTable); + onTableChanged(olapTable); } finally { olapTable.writeUnlock(); } @@ -7221,7 +7227,7 @@ private void setReplicaVersionInternal(long tabletId, long backendId, Long versi LOG.info("set replica {} of tablet {} on backend {} as version {}, last success version {}, " + "last failed version {}, update time {}. is replay: {}", replica.getId(), tabletId, backendId, version, lastSuccessVersion, lastFailedVersion, updateTime, isReplay); - Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(table); + onTableChanged(table); } finally { table.writeUnlock(); } @@ -7302,7 +7308,7 @@ public int setPartitionVersionInternal(String database, String table, long parti + " {}.", partitionId, oldVersion, visibleVersion, database, table, isReplay); } - Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(olapTable); + onTableChanged(olapTable); } finally { olapTable.writeUnlock(); } @@ -7609,6 +7615,86 @@ public NereidsSortedPartitionsCacheManager getSortedPartitionsCacheManager() { return sortedPartitionsCacheManager; } + /** Invalidate derived query caches after applying an existing table metadata journal. */ + public void onTableChanged(TableIf table) { + onTableChanged(table, table.getName()); + } + + /** Notify after mutation while retaining the original name of a renamed or replaced table. */ + public void onTableChanged(TableIf table, String originalName) { + DatabaseIf db = table.getDatabase(); + FullTableName name = db == null || db.getCatalog() == null ? null + : new FullTableName(db.getCatalog().getName(), db.getFullName(), originalName); + onTableChanged(table.getId(), name, table instanceof OlapTable); + } + + /** The table may have been removed by the operation whose journal is being replayed. */ + public void onTableChanged(long dbId, long tableId) { + Database db = getInternalCatalog().getDbNullable(dbId); + Table table = db == null ? null : db.getTableNullable(tableId); + if (table != null) { + onTableChanged(table); + } else { + onTableChanged(tableId, null, true); + } + } + + private void onTableChanged(long tableId, FullTableName name, boolean olapTable) { + invalidateCache("SQL", () -> getSqlCacheManager().invalidateAboutTable(tableId, name)); + invalidateCache("sorted partitions", () -> { + getSortedPartitionsCacheManager().invalidateTable(tableId); + if (name != null) { + getSortedPartitionsCacheManager().invalidateTable(name.catalog, name.db, name.table); + } + }); + if (name != null) { + invalidateCache("MTMV rewrite", () -> getMtmvService().getRelationManager() + .invalidateRewriteCaches(name.catalog, name.db, name.table)); + } + if (olapTable) { + invalidateCache("simple aggregates", () -> SimpleAggCacheMgr.internalInstance().invalidateTable(tableId)); + } + } + + /** Name-based external refresh logs can arrive before a local table object has been loaded. */ + public void onTableChanged(String catalog, String db, String table) { + if (Strings.isNullOrEmpty(catalog) || InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalog)) { + Database database = getInternalCatalog().getDbNullable(db); + Table current = database == null ? null : database.getTableNullable(table); + if (current != null) { + onTableChanged(current); + return; + } + catalog = InternalCatalog.INTERNAL_CATALOG_NAME; + } + onTableChanged(-1L, new FullTableName(catalog, db, table), false); + } + + public void onDatabaseChanged(String catalog, String db) { + invalidateCache("SQL", () -> getSqlCacheManager().invalidateAboutDatabase(catalog, db)); + invalidateCache("sorted partitions", () -> getSortedPartitionsCacheManager().invalidateDatabase(catalog, db)); + invalidateCache("MTMV rewrite", () -> getMtmvService().getRelationManager() + .invalidateRewriteCaches(catalog, db, null)); + } + + public void onCatalogChanged(String catalog) { + invalidateCache("SQL", () -> getSqlCacheManager().invalidateAboutCatalog(catalog)); + invalidateCache("sorted partitions", () -> getSortedPartitionsCacheManager().invalidateCatalog(catalog)); + invalidateCache("MTMV rewrite", () -> getMtmvService().getRelationManager() + .invalidateRewriteCaches(catalog, null, null)); + } + + private void invalidateCache(String cache, Runnable invalidation) { + if (isCheckpointThread()) { + return; + } + try { + invalidation.run(); + } catch (Throwable t) { + LOG.warn("Failed to invalidate {} cache after applying a metadata change", cache, t); + } + } + public SplitSourceManager getSplitSourceManager() { return splitSourceManager; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index fe07e66bcab5a6..325838a2549d1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -69,6 +69,7 @@ public class MTMV extends OlapTable { private static final Logger LOG = LogManager.getLogger(MTMV.class); + private static final int MAX_REWRITE_CACHE_BUILD_ATTEMPTS = 4; private ReentrantReadWriteLock mvRwLock; @SerializedName("ri") @@ -489,7 +490,7 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws boolean sessionVarsMatch = SessionVarGuardRewriter.checkSessionVariablesMatch( currentSessionVars, this.sessionVariables); - while (true) { + for (int attempt = 0; attempt < MAX_REWRITE_CACHE_BUILD_ATTEMPTS; attempt++) { long cacheGeneration; readMvLock(); try { @@ -524,6 +525,9 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws writeMvUnlock(); } } + // MV rewriting and derived traits are optional; callers already skip them on analysis failure. + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "Failed to build rewrite cache for MV " + getName() + " because metadata kept changing"); } /** Invalidate both cache variants and reject any older build still in progress. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index ef9a823fd01e31..8f4dac01d43cd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -321,6 +321,7 @@ default void addUniqueConstraint(String name, ImmutableList columns, boo UniqueConstraint uniqueConstraint = new UniqueConstraint(name, ImmutableSet.copyOf(columns)); checkConstraintNotExistenceUnsafe(name, uniqueConstraint, constraintMap); constraintMap.put(name, uniqueConstraint); + notifyConstraintChanged(getConstraintRelatedTables(uniqueConstraint)); if (!replay) { Env.getCurrentEnv().getEditLog().logAddConstraint( new AlterConstraintLog(uniqueConstraint, this)); @@ -332,6 +333,7 @@ default void addPrimaryKeyConstraint(String name, ImmutableList columns, PrimaryKeyConstraint primaryKeyConstraint = new PrimaryKeyConstraint(name, ImmutableSet.copyOf(columns)); checkConstraintNotExistenceUnsafe(name, primaryKeyConstraint, constraintMap); constraintMap.put(name, primaryKeyConstraint); + notifyConstraintChanged(getConstraintRelatedTables(primaryKeyConstraint)); if (!replay) { Env.getCurrentEnv().getEditLog().logAddConstraint( new AlterConstraintLog(primaryKeyConstraint, this)); @@ -363,6 +365,7 @@ default void addForeignConstraint(String name, ImmutableList columns, referencedTable); primaryKeyConstraint.addForeignTable(this); constraintMap.put(name, foreignKeyConstraint); + notifyConstraintChanged(getConstraintRelatedTables(foreignKeyConstraint)); if (!replay) { Env.getCurrentEnv().getEditLog().logAddConstraint( new AlterConstraintLog(foreignKeyConstraint, this)); @@ -407,6 +410,7 @@ default void dropConstraintRefWithLock(Constraint constraint) { MetaLockUtils.writeLockTables(tables); try { dropConstraintRef(constraint); + notifyConstraintChanged(tables); } finally { MetaLockUtils.writeUnlockTables(tables); } @@ -441,6 +445,11 @@ default List getConstraintRelatedTables(Constraint constraint) { return tables; } + default void notifyConstraintChanged(List relatedTables) { + Env.getCurrentEnv().onTableChanged(this); + relatedTables.forEach(table -> Env.getCurrentEnv().onTableChanged(table)); + } + default void dropConstraint(String name, boolean replay) { Map constraintMap = getConstraintsMapUnsafe(); if (!constraintMap.containsKey(name)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java index 2d670aa011ead7..8d4ed545c60c0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java @@ -71,6 +71,19 @@ public void invalidateTable(String catalog, String db, String table) { partitionCaches.invalidate(key); } + /** Match the stable ID as well, including entries under a table's previous name. */ + public void invalidateTable(long tableId) { + partitionCaches.asMap().entrySet().removeIf(entry -> entry.getValue().tableId == tableId); + } + + public void invalidateDatabase(String catalog, String db) { + partitionCaches.asMap().keySet().removeIf(key -> key.catalog.equals(catalog) && key.db.equals(db)); + } + + public void invalidateCatalog(String catalog) { + partitionCaches.asMap().keySet().removeIf(key -> key.catalog.equals(catalog)); + } + public Optional> get( SupportBinarySearchFilteringPartitions table, CatalogRelation scan) { ConnectContext connectContext = ConnectContext.get(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java index 31dd8bb170566f..b704a4e5978637 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java @@ -93,6 +93,9 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.stream.Stream; /** * NereidsSqlCacheManager @@ -102,6 +105,7 @@ public class NereidsSqlCacheManager { // key: :: // value: SqlCacheContext private volatile Cache sqlCaches; + private final SqlCacheMetadataVersions metadataVersions = new SqlCacheMetadataVersions(); public NereidsSqlCacheManager() { sqlCaches = buildSqlCaches( @@ -110,13 +114,16 @@ public NereidsSqlCacheManager() { ); } + public SqlCacheMetadataVersions.Snapshot newMetadataSnapshot() { + return metadataVersions.snapshot(); + } + @VisibleForTesting public Cache getSqlCaches() { return sqlCaches; } public void invalidateAboutTable(TableIf tableIf) { - Set invalidateKeys = new LinkedHashSet<>(); FullTableName invalidateTableName = null; DatabaseIf database = tableIf.getDatabase(); if (database != null) { @@ -128,32 +135,50 @@ public void invalidateAboutTable(TableIf tableIf) { } } - for (Entry kv : sqlCaches.asMap().entrySet()) { - String key = kv.getKey(); - SqlCacheContext context = kv.getValue(); - for (Entry nameToVersion : context.getUsedTables().entrySet()) { - FullTableName tableName = nameToVersion.getKey().tableName; - TableVersion tableVersion = nameToVersion.getValue(); - if (tableVersion.id == tableIf.getId()) { - invalidateKeys.add(key); - break; - } - if (tableName.equals(invalidateTableName)) { - invalidateKeys.add(key); - break; - } - } - } + invalidateAboutTable(tableIf.getId(), invalidateTableName); + } - for (String invalidateKey : invalidateKeys) { - sqlCaches.invalidate(invalidateKey); - } + /** Also accepts the identity of a table that has already been dropped or renamed. */ + public void invalidateAboutTable(long tableId, FullTableName invalidateTableName) { + metadataVersions.invalidateTable(tableId, invalidateTableName); + invalidateEntries(context -> context.getUsedTables().entrySet().stream() + .anyMatch(entry -> entry.getValue().id == tableId + || entry.getKey().tableName.equals(invalidateTableName)) + || context.getUsedViews().containsKey(invalidateTableName)); } public void invalidateAll() { + metadataVersions.invalidateAll(); sqlCaches.invalidateAll(); } + public void invalidateAboutDatabase(String catalog, String db) { + invalidateAboutNamespace(catalog, db); + } + + public void invalidateAboutCatalog(String catalog) { + invalidateAboutNamespace(catalog, null); + } + + private void invalidateAboutNamespace(String catalog, String db) { + metadataVersions.invalidateNamespace(catalog, db); + invalidateEntries(context -> Stream.concat( + context.getUsedTables().keySet().stream().map(reference -> reference.tableName), + context.getUsedViews().keySet().stream()) + .anyMatch(name -> name.catalog.equals(catalog) && (db == null || name.db.equals(db)))); + } + + private void invalidateEntries(Predicate affected) { + Set invalidateKeys = new LinkedHashSet<>(); + for (Entry entry : sqlCaches.asMap().entrySet()) { + SqlCacheContext context = entry.getValue(); + if (affected.test(context)) { + invalidateKeys.add(entry.getKey()); + } + } + sqlCaches.invalidateAll(invalidateKeys); + } + public static synchronized void updateConfig() { Env currentEnv = Env.getCurrentEnv(); if (currentEnv == null) { @@ -215,7 +240,7 @@ public void tryAddFeSqlCache(ConnectContext connectContext, String sql) { if (sqlCaches.getIfPresent(key) == null && sqlCacheContext.getOrComputeCacheKeyMd5(sessionVariable) != null && sqlCacheContext.getResultSetInFe().isPresent()) { sqlCacheContext.setAffectQueryResultVariables(sessionVariable); - sqlCaches.put(key, sqlCacheContext); + publishIfDependenciesValid(connectContext, key, sqlCacheContext); } } @@ -265,7 +290,7 @@ public void tryAddBeCache(ConnectContext connectContext, String sql, CacheAnalyz return; } - sqlCaches.put(key, sqlCacheContext); + publishIfDependenciesValid(connectContext, key, sqlCacheContext); } } @@ -346,24 +371,9 @@ private Optional tryParseSql( if (privilegeChanged(connectContext, env, sqlCacheContext)) { return invalidateCache(key); } - IsChanged tableIsChanged = tablesOrDataChanged(env, sqlCacheContext, connectContext.getStatementContext()); - if (tableIsChanged.changed) { - if (tableIsChanged.invalidCache) { - return invalidateCache(key); - } else { - return Optional.empty(); - } - } - IsChanged viewIsChanged = viewsChanged(env, sqlCacheContext); - if (viewIsChanged.changed) { - if (viewIsChanged.invalidCache) { - return invalidateCache(key); - } else { - return Optional.empty(); - } - } - if (externalCatalogChanged(env, sqlCacheContext)) { - return invalidateCache(key); + IsChanged dependencies = dependenciesChanged(env, sqlCacheContext, connectContext.getStatementContext()); + if (dependencies.changed) { + return dependencies.invalidCache ? invalidateCache(key) : Optional.empty(); } LogicalEmptyRelation whateverPlan = new LogicalEmptyRelation(new RelationId(0), ImmutableList.of()); @@ -432,8 +442,46 @@ private Optional tryParseSql( } } + private void publishIfDependenciesValid(ConnectContext context, String key, SqlCacheContext candidate) { + // Validate against fresh MVCC snapshots instead of the executing statement's retained metadata. + StatementContext executing = context.getStatementContext(); + try (StatementContext validation = new StatementContext(context, null)) { + context.setStatementContext(validation); + boolean changed = dependenciesChanged(context.getEnv(), candidate, validation).changed; + if (!changed) { + sqlCaches.put(key, candidate); + } + } catch (Throwable t) { + LOG.warn("Skip publishing SQL cache for query {} because dependency validation failed", + candidate.getQueryId(), t); + } finally { + context.setStatementContext(executing); + } + } + + // DML publishes table/partition versions; validate them here on admission and lookup instead of scanning + // cached queries while transaction publication or replay holds table write locks. Metadata journals also + // advance the independent revision fence because their mutations need not change data versions. + private IsChanged dependenciesChanged(Env env, SqlCacheContext context, StatementContext statement) { + if (!context.metadataVersionsMatch()) { + return IsChanged.CHANGED_AND_INVALIDATE_CACHE; + } + IsChanged tables = tablesOrDataChanged(env, context, statement); + if (tables.changed) { + return tables; + } + IsChanged views = viewsChanged(env, context); + if (views.changed) { + return views; + } + if (externalCatalogChanged(env, context) || !context.metadataVersionsMatch()) { + return IsChanged.CHANGED_AND_INVALIDATE_CACHE; + } + return IsChanged.NOT_CHANGED; + } + private IsChanged tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext, StatementContext statement) { - if (sqlCacheContext.hasUnsupportedTables()) { + if (!sqlCacheContext.supportSqlCache()) { return IsChanged.CHANGED_AND_INVALIDATE_CACHE; } @@ -457,6 +505,9 @@ private IsChanged tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext, } if (tableIf instanceof OlapTable) { OlapTable olapTable = (OlapTable) tableIf; + if (olapSchemaChanged(olapTable, tableVersion)) { + return IsChanged.CHANGED_AND_INVALIDATE_CACHE; + } long currentTableVersion = 0L; try { currentTableVersion = olapTable.getVisibleVersion(); @@ -523,6 +574,19 @@ private IsChanged tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext, return IsChanged.NOT_CHANGED; } + private boolean olapSchemaChanged(OlapTable table, TableVersion version) { + if (!table.tryReadLock(1, TimeUnit.MINUTES)) { + return true; + } + try { + return version.schemaVersion >= 0 + && (version.baseIndexId != table.getBaseIndexId() + || version.schemaVersion != table.getBaseSchemaVersion()); + } finally { + table.readUnlock(); + } + } + private IsChanged viewsChanged(Env env, SqlCacheContext sqlCacheContext) { for (Entry cacheView : sqlCacheContext.getUsedViews().entrySet()) { TableIf currentView = findTableIf(env, cacheView.getKey()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/SqlCacheMetadataVersions.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/SqlCacheMetadataVersions.java new file mode 100644 index 00000000000000..d1b42c1db73443 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/SqlCacheMetadataVersions.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.cache; + +import org.apache.doris.nereids.SqlCacheContext.FullTableName; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** Process-local metadata revisions shared by SQL cache dependencies, independent of data versions. */ +public final class SqlCacheMetadataVersions { + private final AtomicLong changes = new AtomicLong(); + private final AtomicLong all = new AtomicLong(); + // A dependency snapshot retains its revision cells. Unused scopes can be collected without tombstones. + private final Cache tableIds = Caffeine.newBuilder().weakValues().build(); + private final Cache names = Caffeine.newBuilder().weakValues().build(); + + public Snapshot snapshot() { + return new Snapshot(); + } + + public void invalidateTable(long tableId, FullTableName name) { + long revision = changes.incrementAndGet(); + if (tableId != -1L) { + advance(tableIds.getIfPresent(tableId), revision); + } + if (name != null) { + advance(names.getIfPresent(name), revision); + } + } + + public void invalidateNamespace(String catalog, String database) { + long revision = changes.incrementAndGet(); + advance(names.getIfPresent(new FullTableName(catalog, database, null)), revision); + } + + public void invalidateAll() { + all.accumulateAndGet(changes.incrementAndGet(), Math::max); + } + + private static void advance(AtomicLong version, long revision) { + // Name-only/cold logs need not create revision cells when no query depends on that scope. + if (version != null) { + version.accumulateAndGet(revision, Math::max); + } + } + + /** One query's captured revisions; no query/context is registered with the revision owner. */ + public final class Snapshot { + private final long start = changes.get(); + private final long allVersion = all.get(); + private final Map dependencies = new LinkedHashMap<>(); + private boolean capturedBeforeChange = true; + + public void addTable(long tableId, FullTableName name) { + // A refresh may retire an object before binding records its first dependency. Do not attach + // a new revision to metadata the statement could have resolved before that refresh. + capturedBeforeChange &= start == changes.get(); + capture(tableIds.get(tableId, ignored -> new AtomicLong())); + capture(names.get(name, ignored -> new AtomicLong())); + capture(names.get(new FullTableName(name.catalog, name.db, null), ignored -> new AtomicLong())); + capture(names.get(new FullTableName(name.catalog, null, null), ignored -> new AtomicLong())); + capturedBeforeChange &= start == changes.get(); + } + + private void capture(AtomicLong version) { + dependencies.putIfAbsent(version, version.get()); + } + + public boolean isCurrent() { + if (!capturedBeforeChange || allVersion != all.get()) { + return false; + } + for (Map.Entry dependency : dependencies.entrySet()) { + if (dependency.getKey().get() != dependency.getValue()) { + return false; + } + } + return true; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 4481d0f7259f6c..0bb6d16853e0f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -191,6 +191,7 @@ private void cleanupRemovedCatalog(RemovedCatalog removedCatalog, boolean perman Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalog(removedCatalog.catalogId); } Env.getCurrentEnv().getQueryStats().clear(removedCatalog.catalogId); + Env.getCurrentEnv().onCatalogChanged(removedCatalog.catalogName); LOG.info("Removed catalog with id {}, name {}", removedCatalog.catalogId, removedCatalog.catalogName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 204574d359501a..b667ead3033d46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -761,7 +761,11 @@ public void resetToUninitialized(boolean invalidCache) { Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(id); } } finally { - endTableLookupRefresh(); + try { + Env.getCurrentEnv().onCatalogChanged(name); + } finally { + endTableLookupRefresh(); + } } } @@ -1463,9 +1467,16 @@ boolean shouldInvalidateRoutedCacheOnDatabaseRemoval() { void handleDatabaseMetaCacheRemoval(Optional> value, RemovalCause cause) { - value.ifPresent(v -> v.resetMetaToUninitialized( - shouldInvalidateRoutedCacheOnDatabaseRemoval(), - !cause.wasEvicted() && shouldInvalidateRowCountOnDatabaseRemoval())); + value.ifPresent(database -> { + if (cause.wasEvicted()) { + // MetaCache objects are local representatives. Capacity/expiry retirement is not + // a schema or data change and must not invalidate a query that is loading its DBs. + database.resetMetaToUninitializedOnEviction(shouldInvalidateRoutedCacheOnDatabaseRemoval()); + } else { + database.resetMetaToUninitialized(shouldInvalidateRoutedCacheOnDatabaseRemoval(), + shouldInvalidateRowCountOnDatabaseRemoval()); + } + }); } /** @@ -1771,7 +1782,7 @@ private void invalidatePropertySpecificCacheGroups(Map updatedPr ExternalMetaCacheMgr extMetaCacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); if (java.util.Objects.nonNull(schemaCacheTtl) || updatedProps.containsKey(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY)) { - extMetaCacheMgr.removeCatalog(id); + extMetaCacheMgr.removeCatalog(id, name); return; } for (String key : updatedProps.keySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 76c65f04abbe6a..4202e8c212638c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -128,6 +128,22 @@ public void resetMetaToUninitialized(boolean invalidateEngineCache) { } public void resetMetaToUninitialized(boolean invalidateEngineCache, boolean invalidateRowCountCache) { + resetMetaToUninitialized(invalidateRowCountCache, () -> { + if (invalidateEngineCache) { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(this); + } + }); + } + + void resetMetaToUninitializedOnEviction(boolean invalidateEngineCache) { + resetMetaToUninitialized(false, () -> { + if (invalidateEngineCache) { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbMetadataCache(this); + } + }); + } + + private void resetMetaToUninitialized(boolean invalidateRowCountCache, Runnable engineInvalidation) { if (LOG.isDebugEnabled()) { LOG.debug("resetToUninitialized db name {}, id {}, isInitializing: {}, initialized: {}", this.name, this.id, isInitializing, initialized, new Exception()); @@ -156,12 +172,8 @@ public void resetMetaToUninitialized(boolean invalidateEngineCache, boolean inva } } try { - if (invalidateEngineCache) { - // Route through the typed overload: connector-specific caches (for example Paimon's - // table loader) are keyed by the database object and are not fully covered by the - // name-based scan in invalidateDb(long, String). - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(this); - } + // Keep typed database identity for connector caches keyed by the retired object. + engineInvalidation.run(); } finally { if (invalidateRowCountCache) { // Independent of the routed invalidation: a connector cache failure (for example diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 91266c74747e7f..15d28200ff866e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -395,22 +395,40 @@ private Map runtimeEffectiveCacheProperties( return effective; } - public void invalidateCatalog(long catalogId) { - rowCountCache.invalidateCatalog(catalogId); + private void invalidateCatalogCaches(long catalogId, String catalogName, Runnable invalidation) { try { + invalidation.run(); + } finally { + try { + if (catalogName != null) { + Env.getCurrentEnv().onCatalogChanged(catalogName); + } else { + Env.getCurrentEnv().getSortedPartitionsCacheManager().invalidateAll(); + } + } finally { + rowCountCache.invalidateCatalog(catalogId); + } + } + } + + public void invalidateCatalog(long catalogId) { + CatalogIf owner = getCatalog(catalogId); + invalidateCatalogCaches(catalogId, owner == null ? null : owner.getName(), () -> { + rowCountCache.invalidateCatalog(catalogId); invalidateLanceTableAccess(catalogId); routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateCatalog", () -> cache.invalidateCatalogEntries(catalogId))); - } finally { - rowCountCache.invalidateCatalog(catalogId); - } + }); } public void invalidateCatalogByEngine(long catalogId, String engine) { - routeSpecifiedEngine(engine, cache -> safeInvalidate( - cache, catalogId, "invalidateCatalogByEngine", - () -> cache.invalidateCatalogEntries(catalogId))); + CatalogIf owner = getCatalog(catalogId); + invalidateCatalogCaches(catalogId, owner == null ? null : owner.getName(), () -> { + routeSpecifiedEngine(engine, cache -> safeInvalidate( + cache, catalogId, "invalidateCatalogByEngine", + () -> cache.invalidateCatalogEntries(catalogId))); + }); } /** @@ -429,15 +447,22 @@ public T withCatalogLifecycleLock(long catalogId, Supplier action) { } public void removeCatalog(long catalogId) { - Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); - lifecycleLock.lock(); - try { - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "removeCatalog", - () -> cache.invalidateCatalog(catalogId))); - } finally { - lifecycleLock.unlock(); - } + CatalogIf owner = getCatalog(catalogId); + removeCatalog(catalogId, owner == null ? null : owner.getName()); + } + + public void removeCatalog(long catalogId, String catalogName) { + invalidateCatalogCaches(catalogId, catalogName, () -> { + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "removeCatalog", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } + }); } /** @@ -454,16 +479,11 @@ public void removeCatalog(long catalogId) { * retained runtime, its group) and lets the next statement load a coherent generation. */ public void onCatalogOperationalContextChanged(long catalogId) { - try { - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "onCatalogOperationalContextChanged", - () -> cache.invalidateCatalogEntries(catalogId))); - } finally { - // The committed property change reset the catalog against a new target. Database objects - // already evicted from the smaller metadata cache never run removal callbacks, so publish - // one catalog-wide row-count fence to cover those cold scopes as well. - rowCountCache.invalidateCatalog(catalogId); - } + CatalogIf owner = getCatalog(catalogId); + invalidateCatalogCaches(catalogId, owner == null ? null : owner.getName(), () -> + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "onCatalogOperationalContextChanged", + () -> cache.invalidateCatalogEntries(catalogId)))); } public void removeCatalogPermanently(long catalogId) { @@ -495,36 +515,41 @@ public void removeCatalogPermanently(long catalogId) { /** Restore catalog properties and retire any group initialized from the rejected candidate atomically. */ public void rollbackCatalogProperties(ExternalCatalog catalog, Map oldProperties) { long catalogId = catalog.getId(); - Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); - lifecycleLock.lock(); - try { + CatalogIf owner = getCatalog(catalogId); + invalidateCatalogCaches(catalogId, owner == null ? null : owner.getName(), () -> { + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); try { - catalog.rollBackCatalogProps(oldProperties); - catalog.resetToUninitialized(false); + try { + catalog.rollBackCatalogProps(oldProperties); + catalog.resetToUninitialized(false); + } finally { + // Retire any group initialized from the rejected candidate even when the reset's + // own cleanup (JDBC/Trino/Lance close) throws; otherwise an already-initialized + // group keeps taking the fast path against the old target. + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "rollbackCatalogProperties", + () -> cache.invalidateCatalog(catalogId))); + } } finally { - // Retire any group initialized from the rejected candidate even when the reset's - // own cleanup (JDBC/Trino/Lance close) throws; otherwise an already-initialized - // group keeps taking the fast path against the old target. - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "rollbackCatalogProperties", - () -> cache.invalidateCatalog(catalogId))); + lifecycleLock.unlock(); } - } finally { - rowCountCache.invalidateCatalog(catalogId); - lifecycleLock.unlock(); - } + }); } public void removeCatalogByEngine(long catalogId, String engine) { - Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); - lifecycleLock.lock(); - try { - routeSpecifiedEngine(engine, cache -> safeInvalidate( - cache, catalogId, "removeCatalogByEngine", - () -> cache.invalidateCatalog(catalogId))); - } finally { - lifecycleLock.unlock(); - } + CatalogIf owner = getCatalog(catalogId); + invalidateCatalogCaches(catalogId, owner == null ? null : owner.getName(), () -> { + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeSpecifiedEngine(engine, cache -> safeInvalidate( + cache, catalogId, "removeCatalogByEngine", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } + }); } public void invalidateDb(long catalogId, String dbName) { @@ -550,6 +575,7 @@ void invalidateDb(long catalogId, long dbId, String dbName, boolean invalidateRo } private void invalidateDb(long catalogId, String dbName, OptionalLong dbId, boolean invalidateRowCountCache) { + CatalogIf owner = getCatalog(catalogId); if (invalidateRowCountCache) { invalidateDatabaseRowCount(catalogId, dbId); } @@ -557,8 +583,14 @@ private void invalidateDb(long catalogId, String dbName, OptionalLong dbId, bool routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateDb", () -> cache.invalidateDb(catalogId, dbName))); } finally { - if (invalidateRowCountCache) { - invalidateDatabaseRowCount(catalogId, dbId); + try { + if (owner != null) { + Env.getCurrentEnv().onDatabaseChanged(owner.getName(), dbName); + } + } finally { + if (invalidateRowCountCache) { + invalidateDatabaseRowCount(catalogId, dbId); + } } } } @@ -574,36 +606,37 @@ private void invalidateDatabaseRowCount(long catalogId, OptionalLong dbId) { } public void invalidateDb(ExternalDatabase database) { + long catalogId = database.getCatalog().getId(); + CatalogIf owner = getCatalog(catalogId); + try { + invalidateDbMetadataCache(database); + } finally { + if (owner != null) { + Env.getCurrentEnv().onDatabaseChanged(owner.getName(), database.getFullName()); + } + } + } + + /** Retire engine entries for a local DB object without reporting a metadata change. */ + void invalidateDbMetadataCache(ExternalDatabase database) { long catalogId = database.getCatalog().getId(); routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateDb", () -> cache.invalidateDb(database))); } public void invalidateTable(long catalogId, String dbName, String tableName) { - Runnable rowCountFence = () -> rowCountCache.invalidateCatalog(catalogId); - try { - rowCountFence = resolveTableRowCountFence(catalogId, dbName, getCachedDb(catalogId, dbName), tableName); - rowCountFence.run(); + invalidateTableCaches(catalogId, dbName, tableName, () -> { invalidateLanceTableAccess(catalogId); routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateTable", () -> cache.invalidateTable(catalogId, dbName, tableName))); - } finally { - rowCountFence.run(); - } + }); } public void invalidateTableByEngine(long catalogId, String engine, String dbName, String tableName) { - Runnable rowCountFence = () -> rowCountCache.invalidateCatalog(catalogId); - try { - rowCountFence = resolveTableRowCountFence(catalogId, dbName, getCachedDb(catalogId, dbName), tableName); - rowCountFence.run(); - routeSpecifiedEngine(engine, cache -> safeInvalidate( - cache, catalogId, "invalidateTableByEngine", - () -> cache.invalidateTable(catalogId, dbName, tableName))); - } finally { - rowCountFence.run(); - } + invalidateTableCaches(catalogId, dbName, tableName, () -> routeSpecifiedEngine(engine, cache -> safeInvalidate( + cache, catalogId, "invalidateTableByEngine", + () -> cache.invalidateTable(catalogId, dbName, tableName)))); } private void invalidateLanceTableAccess(long catalogId) { @@ -617,15 +650,27 @@ private void invalidateLanceTableAccess(long catalogId) { public void invalidatePartitions(long catalogId, String dbName, String tableName, List partitions) { + invalidateTableCaches(catalogId, dbName, tableName, () -> + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "invalidatePartitions", + () -> cache.invalidatePartitions(catalogId, dbName, tableName, partitions)))); + } + + private void invalidateTableCaches(long catalogId, String dbName, String tableName, Runnable invalidation) { + CatalogIf owner = getCatalog(catalogId); Runnable rowCountFence = () -> rowCountCache.invalidateCatalog(catalogId); try { rowCountFence = resolveTableRowCountFence(catalogId, dbName, getCachedDb(catalogId, dbName), tableName); rowCountFence.run(); - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "invalidatePartitions", - () -> cache.invalidatePartitions(catalogId, dbName, tableName, partitions))); + invalidation.run(); } finally { - rowCountFence.run(); + try { + if (owner != null) { + Env.getCurrentEnv().onTableChanged(owner.getName(), dbName, tableName); + } + } finally { + rowCountFence.run(); + } } } @@ -841,7 +886,12 @@ public void invalidateTableCache(ExternalTable dorisTable) { routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateTable", () -> cache.invalidateTable(dorisTable))); } finally { - invalidateRowCountCache(dorisTable); + try { + Env.getCurrentEnv().onTableChanged(dorisTable.getCatalog().getName(), + dorisTable.getDb().getFullName(), dorisTable.getName()); + } finally { + invalidateRowCountCache(dorisTable); + } } if (LOG.isDebugEnabled()) { LOG.debug("invalid table cache for {}.{} in catalog {}", dorisTable.getRemoteDbName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index adda42ad5c1498..5e5d560dfe68df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -129,7 +129,6 @@ import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.nereids.stats.SimpleAggCacheMgr; import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand.IdType; import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; import org.apache.doris.persist.AlterDatabasePropertyInfo; @@ -1043,6 +1042,7 @@ public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop, Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(), db.getId(), table.getId()); table.removeTableIdentifierFromPrimaryTable(); db.unregisterTable(table.getId()); + Env.getCurrentEnv().onTableChanged(table); StopWatch watch = StopWatch.createStarted(); Env.getCurrentRecycleBin().recycleTable(db.getId(), table, isReplay, isForceDrop, recycleTime); watch.stop(); @@ -2084,7 +2084,7 @@ public void dropPartitionWithoutCheck(Database db, OlapTable olapTable, String p } } if (partition != null && !isTempPartition) { - SimpleAggCacheMgr.internalInstance().invalidateTable(olapTable.getId()); + Env.getCurrentEnv().onTableChanged(olapTable); } // Here, we only wait for the EventProcessor to finish processing the event, @@ -2127,7 +2127,7 @@ public void replayDropPartition(DropPartitionInfo info) throws MetaNotFoundExcep } } if (partition != null && !info.isTempPartition()) { - SimpleAggCacheMgr.internalInstance().invalidateTable(olapTable.getId()); + Env.getCurrentEnv().onTableChanged(olapTable); } olapTable.updateVisibleVersionAndTime(info.getVersion(), info.getVersionTime()); // Replay set new partition loaded flag to true for auto analyze. @@ -4011,6 +4011,7 @@ private List truncateTableInternal(OlapTable olapTable, List mv1, mv1==>mv2, table1 ==> mv2 // `tableMTMVsOneLevelAndFromView` will have 2 pair: table1 ==> mv1, mv1==>mv2 // `viewMTMVs` will have 2 pair: v1 ==> mv1, v2 ==> mv1 + // Only dependency replacement and invalidation discovery use this lock. Release it before + // resolving tables or taking an MV lock: refresh completion can already hold that MV lock. + private final Object relationLock = new Object(); private final Map> tableMTMVs = Maps.newConcurrentMap(); private final Map> tableMTMVsOneLevelAndFromView = Maps.newConcurrentMap(); // view => mtmv @@ -79,6 +86,52 @@ public Set getMtmvsByBaseView(BaseTableInfo table) { return viewMTMVs.getOrDefault(table, ImmutableSet.of()); } + /** Only discard local rewrite plans; replay must not refresh an MV or load external metadata. */ + public void invalidateRewriteCaches(String catalog, String db, String table) { + Set affected = collectAffectedMTMVs(catalog, db, table); + List targets = Lists.newArrayList(); + for (BaseTableInfo mv : affected) { + Database database = Env.getCurrentInternalCatalog().getDbNullable(mv.getDbName()); + Table current = database == null ? null : database.getTableNullable(mv.getTableName()); + if (current instanceof MTMV) { + targets.add((MTMV) current); + } + } + // The existing generation barrier also rejects plans whose construction was already in flight. + for (MTMV target : targets) { + try { + target.invalidateRewriteCache(); + } catch (Throwable failure) { + LOG.warn("Failed to invalidate rewrite cache for MV {} after table metadata changed: {}.{}.{}", + target.getId(), catalog, db, table, failure); + } + } + } + + private Set collectAffectedMTMVs(String catalog, String db, String table) { + synchronized (relationLock) { + Set affected = Sets.newHashSet(); + if (table != null) { + BaseTableInfo base = new BaseTableInfo(catalog, db, table); + affected.addAll(getMtmvsByBaseTable(base)); + affected.addAll(getMtmvsByBaseView(base)); + if (InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalog)) { + affected.add(base); + } + } else { + for (Map> relations : ImmutableList.of(tableMTMVs, viewMTMVs)) { + for (Map.Entry> entry : relations.entrySet()) { + BaseTableInfo base = entry.getKey(); + if (catalog.equals(base.getCtlName()) && (db == null || db.equals(base.getDbName()))) { + affected.addAll(entry.getValue()); + } + } + } + } + return affected; + } + } + public Set getMtmvsByBaseTableOneLevelAndFromView(BaseTableInfo table) { return tableMTMVsOneLevelAndFromView.getOrDefault(table, ImmutableSet.of()); } @@ -150,30 +203,27 @@ private Set getMTMVInfos(List tableInfos) { } private Set getOrCreateMTMVs(BaseTableInfo baseTableInfo) { - if (!tableMTMVs.containsKey(baseTableInfo)) { - tableMTMVs.put(baseTableInfo, Sets.newConcurrentHashSet()); - } - return tableMTMVs.get(baseTableInfo); + return tableMTMVs.computeIfAbsent(baseTableInfo, ignored -> Sets.newConcurrentHashSet()); } private Set getOrCreateMTMVsView(BaseTableInfo baseTableInfo) { - if (!viewMTMVs.containsKey(baseTableInfo)) { - viewMTMVs.put(baseTableInfo, Sets.newConcurrentHashSet()); - } - return viewMTMVs.get(baseTableInfo); + return viewMTMVs.computeIfAbsent(baseTableInfo, ignored -> Sets.newConcurrentHashSet()); } private Set getOrCreateMTMVsOneLevelAndFromView(BaseTableInfo baseTableInfo) { - if (!tableMTMVsOneLevelAndFromView.containsKey(baseTableInfo)) { - tableMTMVsOneLevelAndFromView.put(baseTableInfo, Sets.newConcurrentHashSet()); - } - return tableMTMVsOneLevelAndFromView.get(baseTableInfo); + return tableMTMVsOneLevelAndFromView.computeIfAbsent(baseTableInfo, ignored -> Sets.newConcurrentHashSet()); } public void refreshMTMVCache(MTMVRelation relation, BaseTableInfo mtmvInfo) { LOG.info("refreshMTMVCache,relation: {}, mtmvInfo: {}", relation, mtmvInfo); - removeMTMV(mtmvInfo); - addMTMV(relation, mtmvInfo); + updateMTMVRelations(relation, mtmvInfo); + } + + private void updateMTMVRelations(MTMVRelation relation, BaseTableInfo mtmvInfo) { + synchronized (relationLock) { + removeMTMV(mtmvInfo); + addMTMV(relation, mtmvInfo); + } } private void addMTMV(MTMVRelation relation, BaseTableInfo mtmvInfo) { @@ -242,7 +292,7 @@ public void registerMTMV(MTMV mtmv, Long dbId) { */ @Override public void unregisterMTMV(MTMV mtmv) { - removeMTMV(new BaseTableInfo(mtmv)); + updateMTMVRelations(null, new BaseTableInfo(mtmv)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java index 11e454b9442246..109c015d32503c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.Pair; +import org.apache.doris.common.cache.SqlCacheMetadataVersions; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.mvcc.MvccReadSelector; @@ -68,6 +69,7 @@ public class SqlCacheContext { private static final Logger LOG = LogManager.getLogger(SqlCacheContext.class); private final UserIdentity userIdentity; + private final SqlCacheMetadataVersions.Snapshot metadataVersions; private volatile TUniqueId queryId; // if contains udf/udaf/tableValuesFunction we can not process it and skip use sql cache private volatile boolean cannotProcessExpression; @@ -113,6 +115,12 @@ public class SqlCacheContext { /** SqlCacheContext */ public SqlCacheContext(UserIdentity userIdentity) { + this(userIdentity, new SqlCacheMetadataVersions().snapshot()); + } + + /** Capture metadata revisions from the SQL cache manager that will validate this statement. */ + public SqlCacheContext(UserIdentity userIdentity, SqlCacheMetadataVersions.Snapshot metadataVersions) { + this.metadataVersions = metadataVersions; if (userIdentity == null) { ConnectContext connectContext = ConnectContext.get(); if (connectContext != null) { @@ -125,6 +133,10 @@ public SqlCacheContext(UserIdentity userIdentity) { this.userIdentity = userIdentity; } + public boolean metadataVersionsMatch() { + return metadataVersions.isCurrent(); + } + public String getPhysicalPlan() { return physicalPlan; } @@ -200,6 +212,8 @@ public synchronized void addUsedTable(TableIf tableIf, Optional sn setCannotProcessExpression(true); return; } + metadataVersions.addTable(tableIf.getId(), + new FullTableName(catalog.getName(), database.getFullName(), tableIf.getName())); if (!catalog.isInternalCatalog() && !this.externalCatalogConfigs.containsKey(catalog.getName())) { this.externalCatalogConfigs.put(catalog.getName(), getCatalogConfigs(catalog)); } @@ -238,7 +252,9 @@ public synchronized void addUsedTable(TableIf tableIf, Optional sn new TableVersion( tableIf.getId(), version, - tableIf.getType(), sqlCacheVersion + tableIf.getType(), sqlCacheVersion, + tableIf instanceof OlapTable ? ((OlapTable) tableIf).getBaseIndexId() : -1L, + tableIf instanceof OlapTable ? ((OlapTable) tableIf).getBaseSchemaVersion() : -1 ) ); } @@ -258,6 +274,8 @@ public synchronized void addUsedView(TableIf tableIf, String ddlSql) { setCannotProcessExpression(true); return; } + metadataVersions.addTable(tableIf.getId(), + new FullTableName(catalog.getName(), database.getFullName(), tableIf.getName())); if (!catalog.isInternalCatalog() && !this.externalCatalogConfigs.containsKey(catalog.getName())) { this.externalCatalogConfigs.put(catalog.getName(), getCatalogConfigs(catalog)); } @@ -482,8 +500,7 @@ public synchronized PUniqueId doComputeCacheKeyMd5( .append("=") .append(tableVersion.id) .append("@") - .append(tableVersion.sqlCacheVersion == null ? Long.toString(tableVersion.version) - : "sql-cache:" + tableVersion.sqlCacheVersion.getVersion()); + .append(tableVersion.getCacheKey()); } for (Entry entry : usedViews.entrySet()) { cacheKey.append("|") @@ -635,10 +652,23 @@ public static class TableVersion { public final long version; public final TableType type; public final SqlCacheVersion sqlCacheVersion; + public final long baseIndexId; + public final int schemaVersion; + + public TableVersion(long id, long version, TableType type, SqlCacheVersion sqlCacheVersion) { + this(id, version, type, sqlCacheVersion, -1L, -1); + } public TableVersion(long id, long version, TableType type) { this(id, version, type, null); } + + /** Data-version and schema identity shared by FE validation and BE cache lookup. */ + public String getCacheKey() { + String dataVersion = sqlCacheVersion == null + ? Long.toString(version) : "sql-cache:" + sqlCacheVersion.getVersion(); + return schemaVersion < 0 ? dataVersion : dataVersion + ":schema:" + baseIndexId + ":" + schemaVersion; + } } /** CacheKeyType */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 7bca37d91b0790..767e360324f2ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -375,9 +375,9 @@ public StatementContext() { this(ConnectContext.get(), null, 0); } - // For StatementScopeIdGenerator + // For StatementScopeIdGenerator; its fallback only allocates IDs and must not retain a connection. public StatementContext(int initialId) { - this(ConnectContext.get(), null, initialId); + this(null, null, initialId); } public StatementContext(ConnectContext connectContext, OriginStatement originStatement) { @@ -395,8 +395,8 @@ private StatementContext(ConnectContext connectContext, OriginStatement originSt if (CacheAnalyzer.canUseSqlCache(connectContext.getSessionVariable())) { // cannot set the queryId here because the queryId for the current query is set // in the subsequent steps. - this.sqlCacheContext = new SqlCacheContext( - connectContext.getCurrentUserIdentity()); + this.sqlCacheContext = new SqlCacheContext(connectContext.getCurrentUserIdentity(), + Env.getCurrentEnv().getSqlCacheManager().newMetadataSnapshot()); if (originStatement != null) { this.sqlCacheContext.setOriginSql(originStatement.originStmt); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java index 1d29fe6ef171ef..5379ccd8dc1f91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java @@ -62,6 +62,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * Rule to bind relations in query plan. @@ -280,27 +281,30 @@ private void collectMTMVCandidates(TableIf table, CascadesContext cascadesContex for (MTMV mtmv : mtmvSet) { cascadesContext.getStatementContext().getCandidateMTMVs().add(mtmv); cascadesContext.getStatementContext().getMtmvRelatedTables().put(mtmv.getFullQualifiers(), mtmv); + List> baseTableQualifiers; mtmv.readMvLock(); try { - for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTables()) { - if (!baseTableInfo.isValid()) { - continue; - } - if (isDebugEnabled) { - LOG.debug("mtmv {} related base table include {}", new BaseTableInfo(mtmv), baseTableInfo); - } - try { - // Collect all base tables and lock them before querying - cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(), - TableFrom.MTMV, Optional.empty()); - } catch (AnalysisException exception) { - LOG.warn("mtmv related base table get err, related table is {}", - baseTableInfo.toList(), exception); - } - } + // Copy names under the MV lock, but resolve external metadata after releasing it: + // catalog retirement holds the catalog monitor while invalidating MV rewrite caches. + baseTableQualifiers = mtmv.getRelation().getBaseTables().stream() + .filter(BaseTableInfo::isValid) + .map(BaseTableInfo::toList) + .collect(Collectors.toList()); } finally { mtmv.readMvUnlock(); } + for (List baseTableQualifier : baseTableQualifiers) { + if (isDebugEnabled) { + LOG.debug("mtmv {} related base table include {}", new BaseTableInfo(mtmv), baseTableQualifier); + } + try { + // Collect all base tables and lock them before querying + cascadesContext.getStatementContext().getAndCacheTable(baseTableQualifier, + TableFrom.MTMV, Optional.empty()); + } catch (AnalysisException exception) { + LOG.warn("mtmv related base table get err, related table is {}", baseTableQualifier, exception); + } + } } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 5ad8af21e38261..f6e396b746da61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -49,6 +49,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.DebugPointUtil; @@ -356,6 +357,8 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_DROP_DB: { DropDbInfo dropDbInfo = (DropDbInfo) journal.getData(); env.replayDropDb(dropDbInfo); + env.onDatabaseChanged(Strings.isNullOrEmpty(dropDbInfo.getCtlName()) + ? InternalCatalog.INTERNAL_CATALOG_NAME : dropDbInfo.getCtlName(), dropDbInfo.getDbName()); break; } case OperationType.OP_ALTER_DB: { @@ -373,6 +376,8 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_RECOVER_DB: { RecoverInfo info = (RecoverInfo) journal.getData(); env.replayRecoverDatabase(info); + env.onDatabaseChanged(InternalCatalog.INTERNAL_CATALOG_NAME, + env.getInternalCatalog().getDbOrMetaException(info.getDbId()).getFullName()); break; } case OperationType.OP_RENAME_DB: { @@ -380,6 +385,8 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { String dbName = dbInfo.getDbName(); LOG.info("Begin to unprotect rename db {}", dbName); env.replayRenameDatabase(dbName, dbInfo.getNewDbName()); + env.onDatabaseChanged(InternalCatalog.INTERNAL_CATALOG_NAME, dbName); + env.onDatabaseChanged(InternalCatalog.INTERNAL_CATALOG_NAME, dbInfo.getNewDbName()); break; } case OperationType.OP_CREATE_TABLE: { @@ -388,8 +395,12 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { env.replayCreateTable(info); if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName().equals( InternalCatalog.INTERNAL_CATALOG_NAME)) { + // Legacy internal journals can omit the top-level table name. + env.onTableChanged(info.getTable()); CreateTableRecord record = new CreateTableRecord(logId, info); env.getBinlogManager().addCreateTableRecord(record); + } else { + env.onTableChanged(info.getCtlName(), info.getDbName(), info.getTblName()); } break; } @@ -398,6 +409,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { LOG.info("Begin to unprotect alter external table schema. db = {} table = {}", info.getDbName(), info.getTableName()); env.replayAlterExternalTableSchema(info.getDbName(), info.getTableName(), info.getNewSchema()); + env.onTableChanged(InternalCatalog.INTERNAL_CATALOG_NAME, info.getDbName(), info.getTableName()); break; } case OperationType.OP_DROP_TABLE: { @@ -414,6 +426,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { .getCatalog(info.getCtl()); if (ctl != null) { ctl.replayDropTable(info.getDb(), info.getTableName(), info.getResolvedDb()); + env.onTableChanged(info.getCtl(), info.getDb(), info.getTableName()); } } break; @@ -425,6 +438,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { + " partitionName = " + info.getPartition().getName()); AddPartitionRecord addPartitionRecord = new AddPartitionRecord(logId, info); env.replayAddPartition(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addAddPartitionRecord(addPartitionRecord); break; } @@ -442,6 +456,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { + info.getTableId() + " partitionId = " + info.getPartitionId()); BatchModifyPartitionsInfo infos = new BatchModifyPartitionsInfo(info); env.getAlterInstance().replayModifyPartition(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addModifyPartitions(infos, logId); break; } @@ -450,6 +465,9 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { for (ModifyPartitionInfo modifyPartitionInfo : info.getModifyPartitionInfos()) { env.getAlterInstance().replayModifyPartition(modifyPartitionInfo); } + info.getModifyPartitionInfos().stream() + .map(partition -> Pair.of(partition.getDbId(), partition.getTableId())) + .distinct().forEach(table -> env.onTableChanged(table.first, table.second)); env.getBinlogManager().addModifyPartitions(info, logId); break; } @@ -466,12 +484,14 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_RECOVER_TABLE: { RecoverInfo info = (RecoverInfo) journal.getData(); env.replayRecoverTable(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addRecoverTableRecord(info, logId); break; } case OperationType.OP_RECOVER_PARTITION: { RecoverInfo info = (RecoverInfo) journal.getData(); env.replayRecoverPartition(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addRecoverTableRecord(info, logId); break; } @@ -484,18 +504,21 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_VIEW_DEF: { AlterViewInfo info = (AlterViewInfo) journal.getData(); env.getAlterInstance().replayModifyViewDef(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addModifyViewDef(info, logId); break; } case OperationType.OP_RENAME_PARTITION: { TableInfo info = (TableInfo) journal.getData(); env.replayRenamePartition(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addPartitionRename(info, logId); break; } case OperationType.OP_RENAME_COLUMN: { TableRenameColumnInfo info = (TableRenameColumnInfo) journal.getData(); env.replayRenameColumn(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addColumnRename(info, logId); break; } @@ -513,6 +536,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_DROP_ROLLUP: { DropInfo info = (DropInfo) journal.getData(); env.getMaterializedViewHandler().replayDropRollup(info, env); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addDropRollup(info, logId); break; } @@ -534,6 +558,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { env.getMaterializedViewHandler().replayDropRollup(info, env); } } + env.onTableChanged(batchDropInfo.getDbId(), batchDropInfo.getTableId()); break; } case OperationType.OP_FINISH_CONSISTENCY_CHECK: { @@ -544,6 +569,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_RENAME_ROLLUP: { TableInfo info = (TableInfo) journal.getData(); env.replayRenameRollup(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addRollupRename(info, logId); break; } @@ -819,6 +845,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_TABLE_COLOCATE: { final TablePropertyInfo info = (TablePropertyInfo) journal.getData(); env.replayModifyTableColocate(info); + env.onTableChanged(info.getDbId(), info.getTableId()); break; } case OperationType.OP_HEARTBEAT: { @@ -994,6 +1021,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_DISTRIBUTION_TYPE: { TableInfo tableInfo = (TableInfo) journal.getData(); env.replayConvertDistributionType(tableInfo); + env.onTableChanged(tableInfo.getDbId(), tableInfo.getTableId()); env.getBinlogManager().addModifyDistributionType(tableInfo, logId); break; } @@ -1010,6 +1038,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { ModifyTableDefaultDistributionBucketNumOperationLog log = (ModifyTableDefaultDistributionBucketNumOperationLog) journal.getData(); env.replayModifyTableDefaultDistributionBucketNum(log); + env.onTableChanged(log.getDbId(), log.getTableId()); env.getBinlogManager().addModifyDistributionNum(log, logId); break; } @@ -1017,6 +1046,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { ReplacePartitionOperationLog replaceTempPartitionLog = (ReplacePartitionOperationLog) journal.getData(); env.replayReplaceTempPartition(replaceTempPartitionLog); + env.onTableChanged(replaceTempPartitionLog.getDbId(), replaceTempPartitionLog.getTblId()); env.getBinlogManager().addReplacePartitions(replaceTempPartitionLog, logId); break; } @@ -1057,6 +1087,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_COMMENT: { ModifyCommentOperationLog operation = (ModifyCommentOperationLog) journal.getData(); env.getAlterInstance().replayModifyComment(operation); + env.onTableChanged(operation.getDbId(), operation.getTblId()); env.getBinlogManager().addModifyComment(operation, logId); break; } @@ -1125,6 +1156,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_TABLE_ENGINE: { ModifyTableEngineOperationLog log = (ModifyTableEngineOperationLog) journal.getData(); env.getAlterInstance().replayProcessModifyEngine(log); + env.onTableChanged(log.getDbId(), log.getTableId()); break; } case OperationType.OP_CREATE_POLICY: { @@ -1175,12 +1207,14 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_TABLE_LIGHT_SCHEMA_CHANGE: { final TableAddOrDropColumnsInfo info = (TableAddOrDropColumnsInfo) journal.getData(); env.getSchemaChangeHandler().replayModifyTableLightSchemaChange(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addModifyTableAddOrDropColumns(info, logId); break; } case OperationType.OP_ALTER_LIGHT_SCHEMA_CHANGE: { final AlterLightSchemaChangeInfo info = (AlterLightSchemaChangeInfo) journal.getData(); env.getSchemaChangeHandler().replayAlterLightSchChange(info); + env.onTableChanged(info.getDbId(), info.getTableId()); break; } case OperationType.OP_CLEAN_QUERY_STATS: { @@ -1192,12 +1226,16 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { final TableAddOrDropInvertedIndicesInfo info = (TableAddOrDropInvertedIndicesInfo) journal.getData(); env.getSchemaChangeHandler().replayModifyTableAddOrDropInvertedIndices(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addModifyTableAddOrDropInvertedIndices(info, logId); break; } case OperationType.OP_INVERTED_INDEX_JOB: { IndexChangeJob indexChangeJob = (IndexChangeJob) journal.getData(); env.getSchemaChangeHandler().replayIndexChangeJob(indexChangeJob); + if (indexChangeJob.getJobState() == IndexChangeJob.JobState.FINISHED) { + env.onTableChanged(indexChangeJob.getDbId(), indexChangeJob.getTableId()); + } env.getBinlogManager().addIndexChangeJob(indexChangeJob, logId); break; } @@ -1387,6 +1425,8 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_ALTER_MTMV: { final AlterMTMV alterMtmv = (AlterMTMV) journal.getData(); env.getAlterInstance().processAlterMTMV(alterMtmv, true); + env.onTableChanged(alterMtmv.getMvName().getCtl(), + alterMtmv.getMvName().getDb(), alterMtmv.getMvName().getTbl()); break; } case OperationType.OP_INSERT_OVERWRITE: { @@ -1455,6 +1495,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { CatalogIf ctl = Env.getCurrentEnv().getCatalogMgr().getCatalog(info.getCtlName()); if (ctl != null) { ctl.replayOperateOnBranchOrTag(info.getDbName(), info.getTblName()); + env.onTableChanged(info.getCtlName(), info.getDbName(), info.getTblName()); } break; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java index ff2772e948200b..8cf0ed41be6860 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java @@ -20,6 +20,7 @@ import org.apache.doris.alter.AlterJobV2.JobState; import org.apache.doris.analysis.AddRollupClause; import org.apache.doris.analysis.AlterClause; +import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.CatalogTestUtil; import org.apache.doris.catalog.Column; @@ -44,6 +45,7 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.info.TableNameInfo; import org.apache.doris.meta.MetaContext; +import org.apache.doris.nereids.SqlCacheContext; import org.apache.doris.nereids.trees.plans.commands.CancelAlterTableCommand; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.OriginStatement; @@ -278,8 +280,14 @@ public void testSchemaChange1() throws Exception { } } + // A query can populate a cache while the shadow index is still invisible. + SqlCacheContext cached = new SqlCacheContext(UserIdentity.ROOT); + cached.addUsedTable(olapTable); + Assert.assertFalse(cached.getUsedTables().isEmpty()); + masterEnv.getSqlCacheManager().getSqlCaches().put("during_rollup", cached); materializedViewHandler.runAfterCatalogReady(); Assert.assertEquals(JobState.FINISHED, rollupJob.getJobState()); + Assert.assertNull(masterEnv.getSqlCacheManager().getSqlCaches().getIfPresent("during_rollup")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/MTMVRewriteCacheInvalidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/MTMVRewriteCacheInvalidationTest.java new file mode 100644 index 00000000000000..2ccd69b9319eaa --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/MTMVRewriteCacheInvalidationTest.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.mtmv.MTMVCache; +import org.apache.doris.mtmv.MTMVRefreshSnapshot; +import org.apache.doris.mtmv.MTMVStatus; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mockito; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +class MTMVRewriteCacheInvalidationTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void repeatedMetadataChangesBoundCacheConstruction(boolean sameSessionVariables) { + AtomicInteger builds = new AtomicInteger(); + MTMVCache stale = Mockito.mock(MTMVCache.class); + MTMV view = new MTMV() { + @Override + protected MTMVCache createRewriteCache(ConnectContext context, boolean needLock, boolean addGuard) { + Assertions.assertTrue(builds.incrementAndGet() <= 8, "cache construction retried indefinitely"); + invalidateRewriteCache(); + return stale; + } + }; + if (!sameSessionVariables) { + Deencapsulation.setField(view, "sessionVariables", ImmutableMap.of("test", "different")); + } + ConnectContext connection = Mockito.mock(ConnectContext.class); + Mockito.when(connection.getSessionVariable()).thenReturn(new SessionVariable()); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> view.getOrGenerateCache(connection)); + Assertions.assertTrue(builds.get() <= 4); + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithGuard")); + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithoutGuard")); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void invalidationRejectsAnOlderBuildForBothCacheVariants(boolean sameSessionVariables) throws Exception { + CountDownLatch firstBuild = new CountDownLatch(1); + CountDownLatch finishBuild = new CountDownLatch(1); + AtomicInteger builds = new AtomicInteger(); + MTMVCache old = Mockito.mock(MTMVCache.class); + MTMVCache fresh = Mockito.mock(MTMVCache.class); + MTMV mv = new MTMV() { + @Override + protected MTMVCache createRewriteCache(ConnectContext context, boolean needLock, boolean addGuard) { + Assertions.assertEquals(!sameSessionVariables, addGuard); + if (builds.incrementAndGet() == 1) { + firstBuild.countDown(); + try { + Assertions.assertTrue(finishBuild.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + return old; + } + return fresh; + } + }; + Deencapsulation.setField(mv, "status", new MTMVStatus()); + Deencapsulation.setField(mv, "refreshSnapshot", new MTMVRefreshSnapshot()); + if (!sameSessionVariables) { + Deencapsulation.setField(mv, "sessionVariables", ImmutableMap.of("test", "different")); + } + ConnectContext connection = Mockito.mock(ConnectContext.class); + Mockito.when(connection.getSessionVariable()).thenReturn(new SessionVariable()); + ExecutorService worker = Executors.newSingleThreadExecutor(); + try { + Future result = worker.submit(() -> mv.getOrGenerateCache(connection)); + Assertions.assertTrue(firstBuild.await(5, TimeUnit.SECONDS)); + mv.invalidateRewriteCache(); + finishBuild.countDown(); + Assertions.assertSame(fresh, result.get(5, TimeUnit.SECONDS)); + Assertions.assertSame(fresh, mv.getOrGenerateCache(connection)); + Assertions.assertEquals(2, builds.get()); + } finally { + finishBuild.countDown(); + worker.shutdownNow(); + Assertions.assertTrue(worker.awaitTermination(5, TimeUnit.SECONDS)); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/cache/SqlCacheMetadataVersionsTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/cache/SqlCacheMetadataVersionsTest.java new file mode 100644 index 00000000000000..2003c44558d64d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/common/cache/SqlCacheMetadataVersionsTest.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.cache; + +import org.apache.doris.nereids.SqlCacheContext.FullTableName; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class SqlCacheMetadataVersionsTest { + private final SqlCacheMetadataVersions versions = new SqlCacheMetadataVersions(); + private final FullTableName table = new FullTableName("catalog", "db", "table"); + + private SqlCacheMetadataVersions.Snapshot snapshot(long id, FullTableName name) { + SqlCacheMetadataVersions.Snapshot snapshot = versions.snapshot(); + snapshot.addTable(id, name); + return snapshot; + } + + @ParameterizedTest + @CsvSource({"id", "name", "database", "catalog", "all"}) + void affectedDependenciesChange(String scope) { + SqlCacheMetadataVersions.Snapshot affected = snapshot(1L, table); + SqlCacheMetadataVersions.Snapshot other = snapshot(2L, new FullTableName("other", "db", "table")); + switch (scope) { + case "id": + versions.invalidateTable(1L, null); + break; + case "name": + versions.invalidateTable(-1L, table); + break; + case "database": + versions.invalidateNamespace("catalog", "db"); + break; + case "catalog": + versions.invalidateNamespace("catalog", null); + break; + case "all": + versions.invalidateAll(); + break; + default: + throw new AssertionError(scope); + } + Assertions.assertFalse(affected.isCurrent()); + Assertions.assertEquals(!scope.equals("all"), other.isCurrent()); + Assertions.assertTrue(snapshot(1L, table).isCurrent()); + } + + @Test + void tableChangePreservesOtherTablesInTheSameNamespace() { + SqlCacheMetadataVersions.Snapshot target = snapshot(1L, table); + SqlCacheMetadataVersions.Snapshot other = snapshot(2L, new FullTableName("catalog", "db", "other")); + versions.invalidateTable(1L, table); + Assertions.assertFalse(target.isCurrent()); + Assertions.assertTrue(other.isCurrent()); + } + + @Test + void dependencyRecordedAfterRefreshCannotAdoptANewRevision() { + SqlCacheMetadataVersions.Snapshot oldQuery = versions.snapshot(); + versions.invalidateNamespace("catalog", "db"); + oldQuery.addTable(1L, table); + Assertions.assertFalse(oldQuery.isCurrent()); + Assertions.assertTrue(snapshot(1L, table).isCurrent()); + } + + @Test + void recordingTheSameTableAgainCannotReviveAnOldDependency() { + SqlCacheMetadataVersions.Snapshot oldQuery = snapshot(1L, table); + versions.invalidateTable(1L, table); + oldQuery.addTable(1L, table); + Assertions.assertFalse(oldQuery.isCurrent()); + } + + @Test + void globalInvalidationIncludesQueriesWithoutTables() { + SqlCacheMetadataVersions.Snapshot constant = versions.snapshot(); + versions.invalidateAll(); + Assertions.assertFalse(constant.isCurrent()); + Assertions.assertTrue(versions.snapshot().isCurrent()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 0fa144a6ef26c9..f631396ea18b47 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -75,7 +75,7 @@ void testDatabaseCapacityEvictionDoesNotFenceAllRowCounts() { ExternalDatabase db = Mockito.mock(ExternalDatabase.class); catalog.handleDatabaseMetaCacheRemoval(Optional.of(db), RemovalCause.SIZE); - Mockito.verify(db).resetMetaToUninitialized(true, false); + Mockito.verify(db).resetMetaToUninitializedOnEviction(true); catalog.handleDatabaseMetaCacheRemoval(Optional.of(db), RemovalCause.EXPLICIT); Mockito.verify(db).resetMetaToUninitialized(true, true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseEvictionQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseEvictionQueryCacheTest.java new file mode 100644 index 00000000000000..ad0e7934d4055b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseEvictionQueryCacheTest.java @@ -0,0 +1,262 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.common.Pair; +import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; +import org.apache.doris.common.cache.NereidsSqlCacheManager; +import org.apache.doris.common.cache.SqlCacheMetadataVersions; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.metacache.ExternalMetaCache; +import org.apache.doris.datasource.metacache.MetaCache; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVCache; +import org.apache.doris.mtmv.MTMVRefreshSnapshot; +import org.apache.doris.mtmv.MTMVRelation; +import org.apache.doris.mtmv.MTMVRelationManager; +import org.apache.doris.mtmv.MTMVService; +import org.apache.doris.mtmv.MTMVStatus; +import org.apache.doris.nereids.SqlCacheContext.FullTableName; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.StatementContext.TableFrom; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; + +class ExternalDatabaseEvictionQueryCacheTest { + @ParameterizedTest + @ValueSource(ints = {1, 2}) + void crossDatabaseCacheBuildSurvivesObjectCapacityPressure(int capacity) { + Fixture fixture = new Fixture(capacity); + AtomicInteger builds = new AtomicInteger(); + MTMVCache generated = Mockito.mock(MTMVCache.class); + MTMV view = new MTMV() { + @Override + protected MTMVCache createRewriteCache(ConnectContext connection, boolean needLock, boolean addGuard) { + // Stop the old unbounded loop deterministically rather than leaking a planner thread. + Assertions.assertTrue(builds.incrementAndGet() <= 8, "MV cache build did not converge"); + try (StatementContext statement = new StatementContext(connection, null)) { + for (String db : ImmutableList.of("one", "two")) { + Assertions.assertNotNull(statement.getAndCacheTable( + ImmutableList.of("lake", db, "t"), TableFrom.QUERY, Optional.empty())); + } + fixture.catalog.objects.getMetaObjCache().cleanUp(); + } + return generated; + } + }; + fixture.setView(view); + try (MockedStatic ignored = fixture.currentEnv()) { + SqlCacheMetadataVersions.Snapshot snapshot = fixture.snapshot(); + Assertions.assertSame(generated, view.getOrGenerateCache(fixture.connection)); + Assertions.assertSame(generated, view.getOrGenerateCache(fixture.connection)); + Assertions.assertEquals(1, builds.get()); + Assertions.assertTrue(snapshot.isCurrent(), "object eviction is not a metadata mutation"); + if (capacity == 1) { + Assertions.assertTrue(fixture.catalog.evictions.get() > 0, "the real DB cache must evict"); + Mockito.verify(fixture.engine, Mockito.atLeastOnce()).invalidateDb(Mockito.any(ExternalDatabase.class)); + } else { + Assertions.assertEquals(0, fixture.catalog.evictions.get()); + } + } + } + + @ParameterizedTest + @EnumSource(value = RemovalCause.class, names = {"SIZE", "EXPIRED", "COLLECTED"}) + void automaticRemovalRetiresMetadataWithoutInvalidatingQueryDependencies(RemovalCause cause) { + Fixture fixture = new Fixture(2); + MTMV view = new MTMV(); + fixture.setView(view); + MTMVCache cached = Mockito.mock(MTMVCache.class); + Deencapsulation.setField(view, "cacheWithGuard", cached); + Deencapsulation.setField(view, "cacheWithoutGuard", cached); + try (MockedStatic ignored = fixture.currentEnv()) { + ExternalDatabase db = fixture.catalog.getDbNullable("one"); + SqlCacheMetadataVersions.Snapshot snapshot = fixture.snapshot(); + + fixture.catalog.handleDatabaseMetaCacheRemoval(Optional.of(db), cause); + + Mockito.verify(fixture.engine).invalidateDb(db); + Assertions.assertTrue(snapshot.isCurrent()); + Assertions.assertSame(cached, Deencapsulation.getField(view, "cacheWithGuard")); + Assertions.assertSame(cached, Deencapsulation.getField(view, "cacheWithoutGuard")); + } + } + + @ParameterizedTest + @EnumSource(value = RemovalCause.class, names = {"EXPLICIT", "REPLACED"}) + void explicitRemovalStillInvalidatesQueryDependencies(RemovalCause cause) { + Fixture fixture = new Fixture(2); + MTMV view = new MTMV(); + fixture.setView(view); + MTMVCache cached = Mockito.mock(MTMVCache.class); + Deencapsulation.setField(view, "cacheWithGuard", cached); + Deencapsulation.setField(view, "cacheWithoutGuard", cached); + try (MockedStatic ignored = fixture.currentEnv()) { + ExternalDatabase db = fixture.catalog.getDbNullable("one"); + SqlCacheMetadataVersions.Snapshot snapshot = fixture.snapshot(); + + fixture.catalog.handleDatabaseMetaCacheRemoval(Optional.of(db), cause); + + Mockito.verify(fixture.engine).invalidateDb(db); + Assertions.assertFalse(snapshot.isCurrent()); + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithGuard")); + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithoutGuard")); + } + } + + private static class Fixture { + private final Env env = Mockito.mock(Env.class, Mockito.CALLS_REAL_METHODS); + private final EvictingCatalog catalog; + private final NereidsSqlCacheManager sql = new NereidsSqlCacheManager(); + private final MTMVRelationManager relations = new MTMVRelationManager(); + private final InternalCatalog internal = Mockito.mock(InternalCatalog.class); + private final Database views = Mockito.mock(Database.class); + private final ExternalMetaCache engine = Mockito.mock(ExternalMetaCache.class); + private final ConnectContext connection = Mockito.mock(ConnectContext.class); + + Fixture(int capacity) { + catalog = new EvictingCatalog(capacity); + CatalogMgr catalogs = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogs).getCatalog("lake"); + Mockito.doReturn(catalog).when(catalogs).getCatalog(catalog.getId()); + Mockito.doReturn(catalogs).when(env).getCatalogMgr(); + Mockito.doReturn(sql).when(env).getSqlCacheManager(); + Mockito.doReturn(Mockito.mock(NereidsSortedPartitionsCacheManager.class)) + .when(env).getSortedPartitionsCacheManager(); + MTMVService service = Mockito.mock(MTMVService.class); + Mockito.when(service.getRelationManager()).thenReturn(relations); + Mockito.doReturn(service).when(env).getMtmvService(); + Mockito.when(internal.getDbNullable("views")).thenReturn(views); + Mockito.when(connection.getEnv()).thenReturn(env); + Mockito.when(connection.getCurrentUserIdentity()).thenReturn(UserIdentity.ROOT); + Mockito.when(connection.getSessionVariable()).thenReturn(new SessionVariable()); + Mockito.when(engine.engine()).thenReturn("default"); + Mockito.when(engine.isCatalogInitialized(catalog.getId())).thenReturn(true); + ExternalMetaCacheMgr caches = new ExternalMetaCacheMgr(true); + caches.replaceEngineCachesForTest(ImmutableList.of(engine)); + Mockito.doReturn(caches).when(env).getExtMetaCacheMgr(); + } + + void setView(MTMV view) { + Deencapsulation.setField(view, "status", new MTMVStatus()); + Deencapsulation.setField(view, "refreshSnapshot", new MTMVRefreshSnapshot()); + Mockito.when(views.getTableNullable("dependent")).thenReturn(view); + ImmutableSet sources = ImmutableSet.of( + new BaseTableInfo("lake", "one", "t"), new BaseTableInfo("lake", "two", "t")); + relations.refreshMTMVCache(new MTMVRelation(sources, sources, sources, + ImmutableSet.of(), ImmutableSet.of()), new BaseTableInfo("internal", "views", "dependent")); + } + + SqlCacheMetadataVersions.Snapshot snapshot() { + SqlCacheMetadataVersions.Snapshot snapshot = sql.newMetadataSnapshot(); + for (String db : ImmutableList.of("one", "two")) { + snapshot.addTable(Util.genIdByName("lake", db, "t"), new FullTableName("lake", db, "t")); + } + return snapshot; + } + + MockedStatic currentEnv() { + MockedStatic current = Mockito.mockStatic(Env.class); + current.when(Env::getCurrentEnv).thenReturn(env); + current.when(Env::getCurrentInternalCatalog).thenReturn(internal); + return current; + } + } + + private static class EvictingCatalog extends ExternalCatalog { + private final MetaCache> objects; + private final AtomicInteger evictions = new AtomicInteger(); + + EvictingCatalog(int capacity) { + super(1L, "lake", InitCatalogLog.Type.TEST, ""); + catalogProperty = new CatalogProperty(null, Collections.emptyMap()); + objects = new MetaCache<>("databases", MoreExecutors.newDirectExecutorService(), + OptionalLong.empty(), OptionalLong.empty(), capacity, + ignored -> ImmutableList.of(Pair.of("one", "one"), Pair.of("two", "two")), + db -> Optional.of(new DummyDatabase(this, db)), + (key, value, cause) -> { + if (cause.wasEvicted()) { + evictions.incrementAndGet(); + } + handleDatabaseMetaCacheRemoval(value, cause); + }); + metaCache = objects; + initialized = true; + } + + @Override + protected List listTableNamesFromRemote(SessionContext context, String db) { + return ImmutableList.of("t"); + } + + @Override + public boolean tableExist(SessionContext context, String db, String table) { + return true; + } + + @Override + protected void initLocalObjectsImpl() { + } + } + + private static class DummyDatabase extends ExternalDatabase { + private final ExternalTable table; + + DummyDatabase(ExternalCatalog catalog, String name) { + super(catalog, Util.genIdByName(catalog.getName(), name), name, name, InitDatabaseLog.Type.TEST); + table = Mockito.mock(ExternalTable.class); + Mockito.when(table.getName()).thenReturn("t"); + Mockito.when(table.getId()).thenReturn(Util.genIdByName(catalog.getName(), name, "t")); + Mockito.doReturn(this).when(table).getDatabase(); + Mockito.doReturn(catalog).when(table).getCatalog(); + } + + @Override + public ExternalTable getTableNullable(String name) { + return table; + } + + @Override + protected ExternalTable buildTableInternal(String remoteName, String localName, long id, + ExternalCatalog catalog, ExternalDatabase database) { + return table; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index 598edb8995f1bc..208e52b66b4ab9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -606,8 +606,10 @@ public void invalidateTable(long id, String dbName, String tableName) { cacheMgr.replaceRowCountCacheForTest(rowCountCache); HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("hms"); ExternalDatabase db = Mockito.mock(ExternalDatabase.class); Mockito.when(db.getId()).thenReturn(dbId); + Mockito.when(db.getFullName()).thenReturn("db"); ExternalTable table = Mockito.mock(ExternalTable.class); Mockito.when(table.getCatalog()).thenReturn(catalog); Mockito.when(table.getDb()).thenReturn(db); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalQueryCacheInvalidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalQueryCacheInvalidationTest.java new file mode 100644 index 00000000000000..baa60d976c104f --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalQueryCacheInvalidationTest.java @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; +import org.apache.doris.common.cache.NereidsSqlCacheManager; +import org.apache.doris.common.cache.SqlCacheMetadataVersions; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.metacache.ExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheRegistry; +import org.apache.doris.datasource.metacache.ExternalMetaCacheRouteResolver; +import org.apache.doris.mtmv.MTMVRelationManager; +import org.apache.doris.mtmv.MTMVService; +import org.apache.doris.nereids.SqlCacheContext.FullTableName; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.concurrent.atomic.AtomicBoolean; + +class ExternalQueryCacheInvalidationTest { + @ParameterizedTest + @CsvSource({"catalog,false", "catalog,true", "remove,false", "remove,true", + "context,false", "context,true", "database,false", "database,true", + "table,false", "table,true", "typed,false", "typed,true", + "engine,false", "engine,true", "partitions,false", "partitions,true"}) + void metadataInvalidationFencesQueryDependenciesAndKeepsTheFinalRowCountFence( + String operation, boolean failure) { + Env env = Mockito.mock(Env.class, Mockito.CALLS_REAL_METHODS); + CatalogMgr catalogs = Mockito.mock(CatalogMgr.class); + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getName()).thenReturn("lake"); + Mockito.when(catalog.getId()).thenReturn(1L); + Mockito.doReturn(catalog).when(catalogs).getCatalog(1L); + Mockito.doReturn(catalogs).when(env).getCatalogMgr(); + NereidsSqlCacheManager sql = new NereidsSqlCacheManager(); + Mockito.doReturn(sql).when(env).getSqlCacheManager(); + Mockito.doReturn(Mockito.mock(NereidsSortedPartitionsCacheManager.class)) + .when(env).getSortedPartitionsCacheManager(); + MTMVRelationManager relations = Mockito.mock(MTMVRelationManager.class); + MTMVService service = Mockito.mock(MTMVService.class); + Mockito.when(service.getRelationManager()).thenReturn(relations); + Mockito.doReturn(service).when(env).getMtmvService(); + SqlCacheMetadataVersions.Snapshot affected = sql.newMetadataSnapshot(); + affected.addTable(3L, new FullTableName("lake", "db", "t")); + SqlCacheMetadataVersions.Snapshot unrelated = sql.newMetadataSnapshot(); + unrelated.addTable(4L, new FullTableName("other", "db", "t")); + ExternalDatabase database = Mockito.mock(ExternalDatabase.class); + Mockito.doReturn(catalog).when(database).getCatalog(); + Mockito.when(database.getId()).thenReturn(2L); + Mockito.when(database.getFullName()).thenReturn("db"); + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.doReturn(catalog).when(table).getCatalog(); + Mockito.doReturn(database).when(table).getDb(); + Mockito.when(table.getId()).thenReturn(3L); + Mockito.when(table.getName()).thenReturn("t"); + + try (MockedStatic current = Mockito.mockStatic(Env.class)) { + current.when(Env::getCurrentEnv).thenReturn(env); + ExternalMetaCacheMgr caches = new ExternalMetaCacheMgr(true); + AtomicBoolean closingFence = new AtomicBoolean(); + ExternalRowCountCache rows = Mockito.mock(ExternalRowCountCache.class, invocation -> { + if (!affected.isCurrent()) { + closingFence.set(true); + } + return Mockito.RETURNS_DEFAULTS.answer(invocation); + }); + Deencapsulation.setField(caches, "rowCountCache", rows); + ExternalMetaCache engine = Mockito.mock(ExternalMetaCache.class, invocation -> { + String method = invocation.getMethod().getName(); + if (method.equals("engine")) { + return "test"; + } + if (method.equals("isCatalogInitialized")) { + return true; + } + if (method.startsWith("invalidate")) { + Assertions.assertTrue(affected.isCurrent(), "engine invalidation precedes derived invalidation"); + if (failure) { + throw new IllegalStateException("engine failure"); + } + } + return Mockito.RETURNS_DEFAULTS.answer(invocation); + }); + ExternalMetaCacheRegistry registry = Mockito.mock(ExternalMetaCacheRegistry.class); + Mockito.when(registry.resolve("test")).thenReturn(engine); + Deencapsulation.setField(caches, "cacheRegistry", registry); + ExternalMetaCacheRouteResolver routes = Mockito.mock(ExternalMetaCacheRouteResolver.class); + Mockito.when(routes.resolveCatalogCaches(1L, catalog)).thenReturn(ImmutableList.of(engine)); + Deencapsulation.setField(caches, "routeResolver", routes); + Runnable invalidate = () -> { + switch (operation) { + case "catalog": + caches.invalidateCatalog(1L); + break; + case "remove": + caches.removeCatalog(1L, "lake"); + break; + case "context": + caches.onCatalogOperationalContextChanged(1L); + break; + case "database": + caches.invalidateDb(1L, 2L, "db"); + break; + case "table": + caches.invalidateTable(1L, "db", "t"); + break; + case "typed": + caches.invalidateTableCache(table); + break; + case "engine": + caches.invalidateTableByEngine(1L, "test", "db", "t"); + break; + case "partitions": + caches.invalidatePartitions(1L, "db", "t", ImmutableList.of("p")); + break; + default: + throw new AssertionError(operation); + } + }; + if (failure) { + Assertions.assertThrows(IllegalStateException.class, invalidate::run); + } else { + invalidate.run(); + } + Assertions.assertFalse(affected.isCurrent()); + Assertions.assertTrue(unrelated.isCurrent()); + Mockito.verify(relations).invalidateRewriteCaches(Mockito.eq("lake"), Mockito.any(), Mockito.any()); + // Every scope must leave row counts fenced after the derived-cache callback completes. + Assertions.assertTrue(closingFence.get(), "row-count invalidation must follow derived invalidation"); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java index 7e549056b2ee7a..01c48e9bb94e00 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; @@ -139,6 +140,7 @@ public void testPausedTvfJobValidatesAlterOffsetAfterRestore() throws Exception ConnectContext previousContext = ConnectContext.get(); try (MockedStatic envMockedStatic = Mockito.mockStatic(Env.class)) { Env env = Mockito.mock(Env.class); + Mockito.when(env.getSqlCacheManager()).thenReturn(new NereidsSqlCacheManager()); InternalCatalog catalog = Mockito.mock(InternalCatalog.class); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); Mockito.when(catalog.getName()).thenReturn(InternalCatalog.INTERNAL_CATALOG_NAME); diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskAuditTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskAuditTest.java index e2f81e4221bb8f..7158dbe8208b7c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskAuditTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskAuditTest.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.StmtType; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Env; +import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.datasource.CatalogMgr; @@ -328,6 +329,7 @@ private AuditEvent runS3Task(RuntimeException commandFailure) throws Exception { private AuditEvent runTask(RuntimeException commandFailure, String sql, SourceOffsetProvider offsetProvider, Offset offset, boolean expectAudit) throws Exception { Env env = Mockito.mock(Env.class); + Mockito.when(env.getSqlCacheManager()).thenReturn(new NereidsSqlCacheManager()); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); InternalCatalog catalog = Mockito.mock(InternalCatalog.class); WorkloadRuntimeStatusMgr statusMgr = Mockito.mock(WorkloadRuntimeStatusMgr.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationCacheInvalidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationCacheInvalidationTest.java new file mode 100644 index 00000000000000..bd287777b7edb7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationCacheInvalidationTest.java @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.info.TableNameInfo; + +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +class MTMVRelationCacheInvalidationTest { + @Test + void tableInvalidationUsesTheExistingGenerationBarrier() { + MTMVRelationManager manager = new MTMVRelationManager(); + BaseTableInfo source = name("lake", "source", "t"); + addDependency(manager, source, "dependent"); + addDependency(manager, name("lake", "source", "other"), "unrelated"); + MTMV dependent = Mockito.mock(MTMV.class); + MTMV unrelated = Mockito.mock(MTMV.class); + try (MockedStatic env = localViews(dependent, unrelated)) { + manager.invalidateRewriteCaches("lake", "source", "t"); + Mockito.verify(dependent).invalidateRewriteCache(); + Mockito.verify(unrelated, Mockito.never()).invalidateRewriteCache(); + } + } + + @Test + void namespaceInvalidationDoesNotRequireWarmBaseObjects() { + MTMVRelationManager manager = new MTMVRelationManager(); + addDependency(manager, name("lake", "first", "t"), "dependent"); + addDependency(manager, name("lake", "second", "t"), "unrelated"); + MTMV first = Mockito.mock(MTMV.class); + MTMV second = Mockito.mock(MTMV.class); + try (MockedStatic env = localViews(first, second)) { + manager.invalidateRewriteCaches("lake", "first", null); + Mockito.verify(first).invalidateRewriteCache(); + Mockito.verify(second, Mockito.never()).invalidateRewriteCache(); + manager.invalidateRewriteCaches("lake", null, null); + Mockito.verify(first, Mockito.times(2)).invalidateRewriteCache(); + Mockito.verify(second).invalidateRewriteCache(); + } + } + + @Test + void oneFailingViewDoesNotBlockTheOtherViews() { + MTMVRelationManager manager = new MTMVRelationManager(); + BaseTableInfo source = name("lake", "source", "t"); + addDependency(manager, source, "dependent"); + addDependency(manager, source, "unrelated"); + MTMV failing = Mockito.mock(MTMV.class); + MTMV healthy = Mockito.mock(MTMV.class); + Mockito.doThrow(new IllegalStateException("cache failure")).when(failing).invalidateRewriteCache(); + try (MockedStatic env = localViews(failing, healthy)) { + manager.invalidateRewriteCaches("lake", "source", "t"); + Mockito.verify(failing).invalidateRewriteCache(); + Mockito.verify(healthy).invalidateRewriteCache(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"tableMTMVs", "viewMTMVs", "tableMTMVsOneLevelAndFromView"}) + void concurrentRegistrationKeepsBothDependentViews(String indexField) throws Exception { + MTMVRelationManager manager = new MTMVRelationManager(); + FirstRegistrationMap index = new FirstRegistrationMap(); + Deencapsulation.setField(manager, indexField, index); + BaseTableInfo source = name("lake", "source", "t"); + Set tables = indexField.equals("viewMTMVs") ? ImmutableSet.of() : ImmutableSet.of(source); + Set views = indexField.equals("viewMTMVs") ? ImmutableSet.of(source) : ImmutableSet.of(); + MTMVRelation relation = new MTMVRelation(tables, tables, tables, views, views); + BaseTableInfo first = name("internal", "views", "dependent"); + BaseTableInfo second = name("internal", "views", "unrelated"); + ExecutorService workers = Executors.newFixedThreadPool(2); + try { + Future firstRegistration = workers.submit(() -> register(manager, relation, first, index)); + Future secondRegistration = workers.submit(() -> register(manager, relation, second, index)); + firstRegistration.get(10, TimeUnit.SECONDS); + secondRegistration.get(10, TimeUnit.SECONDS); + } finally { + workers.shutdownNow(); + Assertions.assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + Assertions.assertEquals(ImmutableSet.of(first, second), index.get(source), + "concurrent first registrations must not replace another MV's dependency set"); + if (!indexField.equals("tableMTMVsOneLevelAndFromView")) { + MTMV firstView = Mockito.mock(MTMV.class); + MTMV secondView = Mockito.mock(MTMV.class); + try (MockedStatic ignored = localViews(firstView, secondView)) { + manager.invalidateRewriteCaches("lake", "source", "t"); + Mockito.verify(firstView).invalidateRewriteCache(); + Mockito.verify(secondView).invalidateRewriteCache(); + } + } + } + + private void register(MTMVRelationManager manager, MTMVRelation relation, + BaseTableInfo view, FirstRegistrationMap index) { + try { + manager.refreshMTMVCache(relation, view); + } finally { + index.firstRegistered.countDown(); + } + } + + private static class FirstRegistrationMap extends ConcurrentHashMap> { + private static final long serialVersionUID = 1L; + private final AtomicInteger misses = new AtomicInteger(); + private final CountDownLatch secondMiss = new CountDownLatch(1); + private final CountDownLatch firstRegistered = new CountDownLatch(1); + + @Override + public boolean containsKey(Object key) { + boolean present = super.containsKey(key); + if (!present) { + // Both old check-then-put callers observe absence. Let one finish registering before + // the other resumes its stale miss. Atomic computeIfAbsent does not take this path. + if (misses.incrementAndGet() == 1) { + await(secondMiss); + } else { + secondMiss.countDown(); + await(firstRegistered); + } + } + return present; + } + + private static void await(CountDownLatch latch) { + try { + Assertions.assertTrue(latch.await(5, TimeUnit.SECONDS), "registration interleaving timed out"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } + } + + private BaseTableInfo name(String catalog, String db, String table) { + return new BaseTableInfo(new TableNameInfo(catalog, db, table)); + } + + private void addDependency(MTMVRelationManager manager, BaseTableInfo source, String view) { + manager.refreshMTMVCache(new MTMVRelation(ImmutableSet.of(source), ImmutableSet.of(source), + ImmutableSet.of(source), ImmutableSet.of(), ImmutableSet.of()), + name("internal", "views", view)); + } + + private MockedStatic localViews(MTMV dependent, MTMV unrelated) { + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + Mockito.when(catalog.getDbNullable("views")).thenReturn(database); + Mockito.when(database.getTableNullable("dependent")).thenReturn(dependent); + Mockito.when(database.getTableNullable("unrelated")).thenReturn(unrelated); + MockedStatic env = Mockito.mockStatic(Env.class); + env.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + return env; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteInvalidationRaceTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteInvalidationRaceTest.java new file mode 100644 index 00000000000000..0f8801155cf5d5 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteInvalidationRaceTest.java @@ -0,0 +1,250 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.info.TableNameInfo; + +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.util.AbstractSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +class MTMVRewriteInvalidationRaceTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void tableInvalidationPreservesCanonicalExternalNames(boolean globalLowerCase) { + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getName()).thenReturn("lake"); + Mockito.when(catalog.getLowerCaseTableNames()).thenReturn(2); + ExternalDatabase database = Mockito.mock(ExternalDatabase.class); + Mockito.doReturn(catalog).when(database).getCatalog(); + Mockito.when(database.getFullName()).thenReturn("source"); + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.doReturn(database).when(table).getDatabase(); + Mockito.when(table.getName()).thenReturn("Orders"); + MTMVRelationManager manager = new MTMVRelationManager(); + MTMV view = new MTMV(); + MTMVCache cached = Mockito.mock(MTMVCache.class); + Deencapsulation.setField(view, "cacheWithGuard", cached); + Deencapsulation.setField(view, "cacheWithoutGuard", cached); + try (MockedStatic env = localView(view)) { + env.when(Env::isStoredTableNamesLowerCase).thenReturn(globalLowerCase); + BaseTableInfo source = new BaseTableInfo(table); + Assertions.assertEquals("Orders", source.getTableName()); + Assertions.assertEquals(2, catalog.getLowerCaseTableNames()); + manager.refreshMTMVCache(relation(source, false), name("internal", "views", "dependent")); + + manager.invalidateRewriteCaches("lake", "source", "Orders"); + + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithGuard")); + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithoutGuard")); + } + } + + @ParameterizedTest + @CsvSource({"table,false", "database,false", "catalog,false", "table,true", "database,true", "catalog,true"}) + void invalidationCannotMissAMaterializedViewDuringDependencyReplacement(String scope, boolean baseView) + throws Exception { + MTMVRelationManager manager = new MTMVRelationManager(); + BaseTableInfo source = name("lake", "source", "t"); + BaseTableInfo viewName = name("internal", "views", "dependent"); + MTMVRelation relation = relation(source, baseView); + manager.refreshMTMVCache(relation, viewName); + Map> index = Deencapsulation.getField(manager, + baseView ? "viewMTMVs" : "tableMTMVs"); + PausingRemovalSet dependencies = new PausingRemovalSet(index.get(source)); + index.put(source, dependencies); + MTMV view = new MTMV(); + MTMVCache cached = Mockito.mock(MTMVCache.class); + AtomicReference updaterThread = new AtomicReference<>(); + AtomicReference invalidatorThread = new AtomicReference<>(); + CountDownLatch invalidationStarted = new CountDownLatch(1); + ExecutorService workers = Executors.newFixedThreadPool(2); + try { + Future update = workers.submit(() -> { + updaterThread.set(Thread.currentThread()); + view.writeMvLock(); + try { + // addTaskResult publishes its plan before replacing the dependency index. + Deencapsulation.setField(view, "cacheWithGuard", cached); + Deencapsulation.setField(view, "cacheWithoutGuard", cached); + manager.refreshMTMVCache(relation, viewName); + } finally { + view.writeMvUnlock(); + } + }); + Assertions.assertTrue(dependencies.removed.await(5, TimeUnit.SECONDS)); + Future invalidate = workers.submit(() -> { + invalidatorThread.set(Thread.currentThread()); + try (MockedStatic ignored = localView(view)) { + invalidationStarted.countDown(); + manager.invalidateRewriteCaches("lake", scope.equals("catalog") ? null : "source", + scope.equals("table") ? "t" : null); + } + }); + Assertions.assertTrue(invalidationStarted.await(5, TimeUnit.SECONDS)); + awaitDiscoveryOrIndexLock(invalidate, invalidatorThread.get(), updaterThread.get()); + dependencies.continueUpdate.countDown(); + update.get(5, TimeUnit.SECONDS); + invalidate.get(5, TimeUnit.SECONDS); + + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithGuard")); + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithoutGuard")); + Assertions.assertEquals(1L, (long) Deencapsulation.getField(view, "rewriteCacheGeneration")); + } finally { + dependencies.continueUpdate.countDown(); + workers.shutdownNow(); + Assertions.assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void cacheInvalidationDoesNotHoldTheDependencyIndexLock() throws Exception { + MTMVRelationManager manager = new MTMVRelationManager(); + BaseTableInfo source = name("lake", "source", "t"); + manager.refreshMTMVCache(relation(source, false), name("internal", "views", "dependent")); + MTMV view = Mockito.mock(MTMV.class); + CountDownLatch cacheInvalidation = new CountDownLatch(1); + CountDownLatch finishInvalidation = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + cacheInvalidation.countDown(); + Assertions.assertTrue(finishInvalidation.await(5, TimeUnit.SECONDS)); + return null; + }).when(view).invalidateRewriteCache(); + ExecutorService workers = Executors.newFixedThreadPool(2); + try { + Future invalidate = workers.submit(() -> { + try (MockedStatic ignored = localView(view)) { + manager.invalidateRewriteCaches("lake", "source", "t"); + } + }); + Assertions.assertTrue(cacheInvalidation.await(5, TimeUnit.SECONDS)); + // Refresh completion can already own an MV write lock. It must still update the index + // while another invalidation waits for that MV lock. + workers.submit(() -> manager.refreshMTMVCache(relation(source, false), + name("internal", "views", "another"))).get(5, TimeUnit.SECONDS); + finishInvalidation.countDown(); + invalidate.get(5, TimeUnit.SECONDS); + } finally { + finishInvalidation.countDown(); + workers.shutdownNow(); + Assertions.assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private static void awaitDiscoveryOrIndexLock(Future discovery, Thread reader, Thread writer) + throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (System.nanoTime() < deadline) { + if (discovery.isDone()) { + // The unfenced implementation finishes the empty lookup before registration resumes. + return; + } + ThreadInfo info = ManagementFactory.getThreadMXBean().getThreadInfo(reader.getId()); + if (info != null && info.getLockOwnerId() == writer.getId()) { + return; + } + TimeUnit.MILLISECONDS.sleep(1); + } + Assertions.fail("invalidation neither completed discovery nor waited for dependency replacement"); + } + + private static class PausingRemovalSet extends AbstractSet { + private final Set entries = ConcurrentHashMap.newKeySet(); + private final CountDownLatch removed = new CountDownLatch(1); + private final CountDownLatch continueUpdate = new CountDownLatch(1); + + PausingRemovalSet(Set entries) { + this.entries.addAll(entries); + } + + @Override + public boolean remove(Object entry) { + boolean changed = entries.remove(entry); + removed.countDown(); + try { + Assertions.assertTrue(continueUpdate.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + return changed; + } + + @Override + public boolean add(BaseTableInfo entry) { + return entries.add(entry); + } + + @Override + public Iterator iterator() { + return entries.iterator(); + } + + @Override + public int size() { + return entries.size(); + } + } + + private static BaseTableInfo name(String catalog, String db, String table) { + return new BaseTableInfo(new TableNameInfo(catalog, db, table)); + } + + private static MTMVRelation relation(BaseTableInfo source, boolean view) { + Set tables = view ? ImmutableSet.of() : ImmutableSet.of(source); + Set views = view ? ImmutableSet.of(source) : ImmutableSet.of(); + return new MTMVRelation(tables, tables, tables, views, views); + } + + private static MockedStatic localView(MTMV view) { + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + Mockito.when(catalog.getDbNullable("views")).thenReturn(database); + Mockito.when(database.getTableNullable("dependent")).thenReturn(view); + MockedStatic env = Mockito.mockStatic(Env.class); + env.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + return env; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java index 3ea04dac288369..b767f929b1a5c2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java @@ -21,8 +21,10 @@ import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; +import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalScanTaskCacheKey; import org.apache.doris.datasource.ExternalTable; @@ -43,6 +45,7 @@ import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; import org.mockito.InOrder; +import org.mockito.MockedStatic; import org.mockito.Mockito; import java.io.Closeable; @@ -63,6 +66,48 @@ public class StatementContextTest { + @Test + public void testIdGeneratorContextDoesNotCaptureCurrentConnection() { + ConnectContext previousContext = ConnectContext.get(); + ConnectContext connection = new ConnectContext(); + connection.setThreadLocalInfo(); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + StatementContext statement = new StatementContext(10000); + org.junit.jupiter.api.Assertions.assertNull(statement.getConnectContext()); + org.junit.jupiter.api.Assertions.assertFalse(statement.getSqlCacheContext().isPresent()); + org.junit.jupiter.api.Assertions.assertEquals(10000, statement.getNextExprId().asInt()); + org.junit.jupiter.api.Assertions.assertEquals(10001, statement.getNextExprId().asInt()); + mockedEnv.verifyNoInteractions(); + } finally { + ConnectContext.remove(); + if (previousContext != null) { + previousContext.setThreadLocalInfo(); + } + } + } + + @Test + public void testSqlCacheSnapshotIsFreshForEachPreparedExecution() { + Env env = Mockito.mock(Env.class); + NereidsSqlCacheManager manager = new NereidsSqlCacheManager(); + Mockito.when(env.getSqlCacheManager()).thenReturn(manager); + ConnectContext connection = new ConnectContext(); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + StatementContext statement = new StatementContext(connection, new OriginStatement("select 1", 0)); + SqlCacheContext firstExecution = statement.getSqlCacheContext().orElseThrow(AssertionError::new); + org.junit.jupiter.api.Assertions.assertTrue(firstExecution.metadataVersionsMatch()); + + manager.invalidateAll(); + org.junit.jupiter.api.Assertions.assertFalse(firstExecution.metadataVersionsMatch()); + SqlCacheContext nextExecution = statement.createNextExecuteContext().getSqlCacheContext() + .orElseThrow(AssertionError::new); + org.junit.jupiter.api.Assertions.assertTrue(nextExecution.metadataVersionsMatch()); + manager.invalidateAll(); + org.junit.jupiter.api.Assertions.assertFalse(nextExecution.metadataVersionsMatch()); + } + } + @Test public void testPreloadedPartitionViewSurvivesPlainHiveBindingAndResets() { StatementContext statement = new StatementContext(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CollectRelationMTMVLockTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CollectRelationMTMVLockTest.java new file mode 100644 index 00000000000000..32c86b1c5ecfe5 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CollectRelationMTMVLockTest.java @@ -0,0 +1,313 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.analysis; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; +import org.apache.doris.common.cache.NereidsSqlCacheManager; +import org.apache.doris.common.cache.SqlCacheMetadataVersions; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalMetaCacheMgr; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.InitCatalogLog; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.SessionContext; +import org.apache.doris.info.TableNameInfo; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVCache; +import org.apache.doris.mtmv.MTMVRelation; +import org.apache.doris.mtmv.MTMVRelationManager; +import org.apache.doris.mtmv.MTMVService; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.SqlCacheContext.FullTableName; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.StatementContext.TableFrom; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +class CollectRelationMTMVLockTest { + @Test + void catalogRetirementCompletesDuringCandidateBaseLookup() throws Exception { + Env env = Mockito.mock(Env.class, Mockito.CALLS_REAL_METHODS); + CatalogMgr catalogs = Mockito.mock(CatalogMgr.class); + LookupCatalog catalog = new LookupCatalog(); + ExternalDatabase database = Mockito.mock(ExternalDatabase.class); + ExternalTable base = Mockito.mock(ExternalTable.class); + Mockito.doReturn(database).when(base).getDatabase(); + Mockito.when(base.getName()).thenReturn("base"); + Mockito.when(base.getId()).thenReturn(3L); + Mockito.doReturn(catalog).when(database).getCatalog(); + Mockito.when(database.getFullName()).thenReturn("source"); + Mockito.doReturn(base).when(database).getTableNullable("base"); + catalog.database = database; + Mockito.doReturn(catalog).when(catalogs).getCatalog("lake"); + Mockito.doReturn(catalog).when(catalogs).getCatalog(1L); + Mockito.doReturn(catalogs).when(env).getCatalogMgr(); + AccessControllerManager access = Mockito.mock(AccessControllerManager.class); + Mockito.doReturn(access).when(env).getAccessManager(); + Mockito.doReturn(new ExternalMetaCacheMgr(true)).when(env).getExtMetaCacheMgr(); + NereidsSqlCacheManager sqlCaches = new NereidsSqlCacheManager(); + Mockito.doReturn(sqlCaches).when(env).getSqlCacheManager(); + Mockito.doReturn(Mockito.mock(NereidsSortedPartitionsCacheManager.class)) + .when(env).getSortedPartitionsCacheManager(); + MTMV mv = view(ImmutableSet.of(name("base"))); + MTMVRelationManager relations = Mockito.spy(new MTMVRelationManager()); + relations.refreshMTMVCache(mv.getRelation(), + new BaseTableInfo(new TableNameInfo("internal", "views", "mv"))); + Mockito.doReturn(ImmutableSet.of(mv)).when(relations).getCandidateMTMVs(Mockito.anyList()); + Mockito.doAnswer(invocation -> { + Assertions.assertFalse(Thread.holdsLock(catalog), "derived invalidation must run outside the catalog monitor"); + return invocation.callRealMethod(); + }).when(relations).invalidateRewriteCaches("lake", null, null); + MTMVService service = Mockito.mock(MTMVService.class); + Mockito.when(service.getRelationManager()).thenReturn(relations); + Mockito.doReturn(service).when(env).getMtmvService(); + InternalCatalog internal = Mockito.mock(InternalCatalog.class); + Database views = Mockito.mock(Database.class); + Mockito.when(internal.getDbNullable("views")).thenReturn(views); + Mockito.when(views.getTableNullable("mv")).thenReturn(mv); + ConnectContext connection = Mockito.mock(ConnectContext.class); + Mockito.when(connection.getEnv()).thenReturn(env); + Mockito.when(connection.getSessionVariable()).thenReturn(new SessionVariable()); + Mockito.when(connection.getCurrentUserIdentity()).thenReturn(UserIdentity.ROOT); + AtomicReference resetFailure = new AtomicReference<>(); + Thread reset = new Thread(() -> { + try (MockedStatic ignored = localEnv(env, internal)) { + Assertions.assertTrue(catalog.lookupStarted.await(5, TimeUnit.SECONDS)); + catalog.resetToUninitialized(false); + } catch (Throwable failure) { + resetFailure.set(failure); + } finally { + catalog.retirementFinished.countDown(); + } + }, "catalog-retirement-during-mv-collection"); + reset.setDaemon(true); + try (MockedStatic ignored = localEnv(env, internal)) { + StatementContext statement = new StatementContext(connection, null); + List qualifier = name("base").toList(); + // QUERY and MTMV have separate lookup maps, even for the same single base table. + Assertions.assertSame(base, statement.getAndCacheTable(qualifier, TableFrom.QUERY, Optional.empty())); + Assertions.assertTrue(statement.getMtmvRelatedTables().isEmpty()); + SqlCacheMetadataVersions.Snapshot snapshot = sqlCaches.newMetadataSnapshot(); + snapshot.addTable(3L, new FullTableName("lake", "source", "base")); + MTMVCache cached = Mockito.mock(MTMVCache.class); + Deencapsulation.setField(mv, "cacheWithGuard", cached); + Deencapsulation.setField(mv, "cacheWithoutGuard", cached); + Assertions.assertSame(cached, Deencapsulation.getField(mv, "cacheWithGuard")); + Assertions.assertSame(cached, Deencapsulation.getField(mv, "cacheWithoutGuard")); + Assertions.assertTrue(snapshot.isCurrent()); + long generation = Deencapsulation.getField(mv, "rewriteCacheGeneration"); + catalog.pauseLookup = true; + reset.start(); + try { + collect(base, statement); + } finally { + reset.join(TimeUnit.SECONDS.toMillis(10)); + } + Assertions.assertFalse(reset.isAlive(), "catalog retirement did not finish"); + Assertions.assertNull(resetFailure.get()); + Assertions.assertFalse(catalog.retirementBlocked.get(), "MV read lock blocked catalog retirement"); + Assertions.assertSame(base, statement.getMtmvRelatedTables().get(qualifier)); + Assertions.assertNull(Deencapsulation.getField(mv, "cacheWithGuard")); + Assertions.assertNull(Deencapsulation.getField(mv, "cacheWithoutGuard")); + Assertions.assertEquals(generation + 1, (long) Deencapsulation.getField(mv, "rewriteCacheGeneration")); + Assertions.assertFalse(snapshot.isCurrent(), "retirement must still fence old SQL dependencies"); + } + } + + @Test + void dependencyNamesAreCopiedBeforeResolvingTables() throws Exception { + BaseTableInfo first = name("first"); + BaseTableInfo second = name("second"); + Set dependencies = new LinkedHashSet<>(Arrays.asList(first, second)); + MTMV mv = view(dependencies); + StatementContext statement = Mockito.spy(new StatementContext(0)); + AtomicReference mutationFailure = new AtomicReference<>(); + Thread mutate = new Thread(() -> { + mv.writeMvLock(); + try { + second.setTableName("changed"); + dependencies.clear(); + } catch (Throwable failure) { + mutationFailure.set(failure); + } finally { + mv.writeMvUnlock(); + } + }, "mv-dependency-update-during-collection"); + mutate.setDaemon(true); + TableIf collected = Mockito.mock(TableIf.class); + Mockito.doAnswer(invocation -> { + mutate.start(); + mutate.join(TimeUnit.SECONDS.toMillis(5)); + Assertions.assertFalse(mutate.isAlive(), "table resolution must not hold the MV lock"); + return collected; + }).when(statement).getAndCacheTable(first.toList(), TableFrom.MTMV, Optional.empty()); + Mockito.doReturn(collected).when(statement) + .getAndCacheTable(second.toList(), TableFrom.MTMV, Optional.empty()); + try { + collectWithCandidate(mv, statement); + } finally { + mutate.join(TimeUnit.SECONDS.toMillis(10)); + } + Assertions.assertNull(mutationFailure.get()); + Mockito.verify(statement).getAndCacheTable(ImmutableList.of("lake", "source", "second"), + TableFrom.MTMV, Optional.empty()); + } + + @Test + void invalidAndMissingDependenciesDoNotPreventCollectingOtherTables() { + BaseTableInfo invalid = name("invalid"); + invalid.setTableName(null); + MTMV mv = view(new LinkedHashSet<>(Arrays.asList(invalid, name("missing"), name("valid")))); + StatementContext statement = Mockito.spy(new StatementContext(0)); + Mockito.doThrow(new AnalysisException("missing table")).when(statement) + .getAndCacheTable(name("missing").toList(), TableFrom.MTMV, Optional.empty()); + Mockito.doReturn(Mockito.mock(TableIf.class)).when(statement) + .getAndCacheTable(name("valid").toList(), TableFrom.MTMV, Optional.empty()); + collectWithCandidate(mv, statement); + Mockito.verify(statement).getAndCacheTable(name("valid").toList(), TableFrom.MTMV, Optional.empty()); + Mockito.verify(statement, Mockito.never()) + .getAndCacheTable(invalid.toList(), TableFrom.MTMV, Optional.empty()); + } + + private void collectWithCandidate(MTMV mv, StatementContext statement) { + Env env = Mockito.mock(Env.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(env.getMtmvService().getRelationManager().getCandidateMTMVs(Mockito.anyList())) + .thenReturn(ImmutableSet.of(mv)); + ExternalTable base = Mockito.mock(ExternalTable.class); + ExternalDatabase database = Mockito.mock(ExternalDatabase.class); + Mockito.doReturn(database).when(base).getDatabase(); + Mockito.doReturn(Mockito.mock(ExternalCatalog.class)).when(database).getCatalog(); + try (MockedStatic ignored = localEnv(env, Mockito.mock(InternalCatalog.class))) { + collect(base, statement); + } + } + + private void collect(TableIf base, StatementContext statement) { + statement.addPlannerHook(InitMaterializationContextHook.INSTANCE); + CascadesContext cascades = Mockito.mock(CascadesContext.class); + Mockito.when(cascades.getStatementContext()).thenReturn(statement); + Deencapsulation.invoke(new CollectRelation(true), "collectMTMVCandidates", base, cascades); + } + + private static MockedStatic localEnv(Env env, InternalCatalog internal) { + MockedStatic scope = Mockito.mockStatic(Env.class); + scope.when(Env::getCurrentEnv).thenReturn(env); + scope.when(Env::getCurrentInternalCatalog).thenReturn(internal); + return scope; + } + + private static BaseTableInfo name(String table) { + return new BaseTableInfo(new TableNameInfo("lake", "source", table)); + } + + private static MTMV view(Set tables) { + MTMV mv = Mockito.spy(new MTMV()); + Deencapsulation.setField(mv, "name", "mv"); + mv.setRelation(relation(tables)); + Database database = Mockito.mock(Database.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Mockito.when(catalog.getName()).thenReturn("internal"); + Mockito.when(database.getFullName()).thenReturn("views"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.doReturn(database).when(mv).getDatabase(); + return mv; + } + + private static MTMVRelation relation(Set tables) { + return new MTMVRelation(tables, tables, tables, ImmutableSet.of(), ImmutableSet.of()); + } + + private static class LookupCatalog extends ExternalCatalog { + private ExternalDatabase database; + private volatile boolean pauseLookup; + private final CountDownLatch lookupStarted = new CountDownLatch(1); + private final CountDownLatch retirementFinished = new CountDownLatch(1); + private final AtomicBoolean retirementBlocked = new AtomicBoolean(); + + LookupCatalog() { + super(1L, "lake", InitCatalogLog.Type.TEST, ""); + initialized = true; + } + + @Override + public ExternalDatabase getDbNullable(String name) { + if (pauseLookup) { + lookupStarted.countDown(); + try { + // Bound the old lock cycle so a failing test releases its MV read lock. + if (!retirementFinished.await(5, TimeUnit.SECONDS)) { + retirementBlocked.set(true); + throw new IllegalStateException("MV collection blocked catalog retirement"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + makeSureInitialized(); + return database; + } + + @Override + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { + return ImmutableList.of(); + } + + @Override + public boolean tableExist(SessionContext ctx, String dbName, String tableName) { + return false; + } + + @Override + protected void initLocalObjectsImpl() { + } + + @Override + public void onRefreshCache(boolean invalidCache) { + initialized = true; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkDisabledTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkDisabledTest.java index 142f35b4da2f83..f9799a21070fca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkDisabledTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkDisabledTest.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.catalog.Env; +import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -59,6 +60,7 @@ void disabledLocalKeepsRemoteSinkValidation() { private Plan analyzeInsert(String sql, boolean authorized) { ConnectContext context = MemoTestUtils.createConnectContext(); Env env = Mockito.mock(Env.class); + Mockito.when(env.getSqlCacheManager()).thenReturn(new NereidsSqlCacheManager()); AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); Mockito.when(env.getAccessManager()).thenReturn(accessManager); Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(authorized); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/AuditLogWorkloadGroupTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/AuditLogWorkloadGroupTest.java index 4cd7dd5dff85b6..accec01867d720 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/AuditLogWorkloadGroupTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/AuditLogWorkloadGroupTest.java @@ -23,6 +23,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ConnectionException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.proto.Data; import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr; @@ -283,6 +284,7 @@ private void runResolveUnderMockEnv(ConnectContext ctx, String userGroup) { private void mockEnvAuth(MockedStatic mockedEnv, String userGroup) { Env env = Mockito.mock(Env.class); + Mockito.when(env.getSqlCacheManager()).thenReturn(new NereidsSqlCacheManager()); Auth auth = Mockito.mock(Auth.class); Mockito.when(env.getAuth()).thenReturn(auth); Mockito.when(auth.getWorkloadGroup(Mockito.anyString())).thenReturn(userGroup); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/MysqlPreparedStatementForwardingTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/MysqlPreparedStatementForwardingTest.java index 0ab8e1eee6f850..9e1cc495e257e7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/MysqlPreparedStatementForwardingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/MysqlPreparedStatementForwardingTest.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MysqlColType; +import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.mysql.MysqlCapability; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.nereids.StatementContext; @@ -55,6 +56,7 @@ public void testForwardedExecutionsRetainTypesAndNullBitmap() throws Exception { follower.setCurrentUserIdentity(UserIdentity.ROOT); follower.setCapability(MysqlCapability.DEFAULT_CAPABILITY); Env env = Mockito.mock(Env.class); + Mockito.when(env.getSqlCacheManager()).thenReturn(new NereidsSqlCacheManager()); Mockito.when(env.getSelfNode()).thenReturn(new SystemInfoService.HostInfo("127.0.0.1", 9010)); PreparedStatementContext prepared = prepare(follower); try (MockedConstruction executors = Mockito.mockConstruction(StmtExecutor.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/TableCacheInvalidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/TableCacheInvalidationTest.java new file mode 100644 index 00000000000000..da15a62e444b57 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/TableCacheInvalidationTest.java @@ -0,0 +1,617 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.View; +import org.apache.doris.catalog.constraint.UniqueConstraint; +import org.apache.doris.common.Config; +import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; +import org.apache.doris.common.cache.NereidsSqlCacheManager; +import org.apache.doris.common.io.Writable; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.journal.JournalEntity; +import org.apache.doris.mtmv.MTMVRelationManager; +import org.apache.doris.mtmv.MTMVService; +import org.apache.doris.nereids.SqlCacheContext; +import org.apache.doris.nereids.SqlCacheContext.FullTableName; +import org.apache.doris.nereids.SqlCacheContext.ScanTable; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.stats.SimpleAggCacheMgr; +import org.apache.doris.persist.AlterConstraintLog; +import org.apache.doris.persist.AlterViewInfo; +import org.apache.doris.persist.CreateTableInfo; +import org.apache.doris.persist.EditLog; +import org.apache.doris.persist.OperationType; +import org.apache.doris.persist.ReplaceTableOperationLog; +import org.apache.doris.persist.SetPartitionVersionOperationLog; +import org.apache.doris.persist.TableInfo; +import org.apache.doris.persist.TableRenameColumnInfo; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.cache.CacheAnalyzer; +import org.apache.doris.qe.cache.SqlCache; +import org.apache.doris.thrift.TUniqueId; +import org.apache.doris.transaction.PartitionCommitInfo; +import org.apache.doris.transaction.TableCommitInfo; +import org.apache.doris.transaction.TransactionState; +import org.apache.doris.transaction.TransactionStatus; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.gson.JsonObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +class TableCacheInvalidationTest extends TestWithFeService { + private final RecordingSimpleAggCache aggregates = new RecordingSimpleAggCache(); + private int sqlCapacity; + private int partitionCapacity; + private Database database; + private OlapTable target; + private OlapTable control; + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("cache_replay"); + } + + @BeforeEach + void prepareTables() throws Exception { + sqlCapacity = Config.sql_cache_manage_num; + partitionCapacity = Config.cache_partition_meta_table_manage_num; + Config.sql_cache_manage_num = 100; + Config.cache_partition_meta_table_manage_num = 100; + NereidsSqlCacheManager.updateConfig(); + NereidsSortedPartitionsCacheManager.updateConfig(); + connectContext.setDatabase("cache_replay"); + connectContext.getSessionVariable().setEnableSqlCache(true); + connectContext.getSessionVariable().cacheSortedPartitionIntervalSecond = -1; + dropTable("cache_replay.target", false); + dropTable("cache_replay.control", false); + dropTable("cache_replay.renamed", false); + dropTable("cache_replay.plain", false); + for (String name : ImmutableList.of("target", "control")) { + createTable("CREATE TABLE cache_replay." + name + " (id INT, v INT) DUPLICATE KEY(id) " + + "PARTITION BY RANGE(id) (PARTITION p1 VALUES LESS THAN (10), " + + "PARTITION p2 VALUES LESS THAN (20)) DISTRIBUTED BY HASH(id) BUCKETS 1 " + + "PROPERTIES ('replication_num'='1')"); + } + createTable("CREATE TABLE cache_replay.plain (id INT, v INT) DUPLICATE KEY(id) " + + "DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES ('replication_num'='1')"); + database = Env.getCurrentInternalCatalog().getDbOrMetaException("cache_replay"); + target = (OlapTable) database.getTableOrMetaException("target"); + control = (OlapTable) database.getTableOrMetaException("control"); + Env.getCurrentEnv().getSqlCacheManager().invalidateAll(); + Env.getCurrentEnv().getSortedPartitionsCacheManager().invalidateAll(); + aggregates.invalidated.clear(); + aggregates.onInvalidate = () -> { }; + SimpleAggCacheMgr.setTestInstance(aggregates); + } + + @AfterEach + void restoreAggregateCache() { + SimpleAggCacheMgr.clearTestInstance(); + Config.sql_cache_manage_num = sqlCapacity; + Config.cache_partition_meta_table_manage_num = partitionCapacity; + NereidsSqlCacheManager.updateConfig(); + NereidsSortedPartitionsCacheManager.updateConfig(); + } + + @ParameterizedTest + @CsvSource({"false", "true"}) + void constraintMutationInvalidatesCachesForLiveAndReplay(boolean replayJournal) throws Exception { + warmBothTables(); + UniqueConstraint constraint = new UniqueConstraint("cache_unique", ImmutableSet.of("id")); + if (replayJournal) { + replay(OperationType.OP_ADD_CONSTRAINT, + new AlterConstraintLog(constraint, target)); + } else { + target.addUniqueConstraint("cache_unique", ImmutableList.of("id"), false); + } + assertTargetInvalidated(); + warmBothTables(); + if (replayJournal) { + replay(OperationType.OP_DROP_CONSTRAINT, + new AlterConstraintLog(constraint, target)); + } else { + target.dropConstraint("cache_unique", false); + } + assertTargetInvalidated(); + } + + @ParameterizedTest + @CsvSource({"false", "true"}) + void removingPrimaryKeyInvalidatesTheReferencingTables(boolean dropPrimaryTable) throws Exception { + target.addPrimaryKeyConstraint("cache_pk", ImmutableList.of("id"), true); + control.addForeignConstraint("cache_fk", ImmutableList.of("id"), target, ImmutableList.of("id"), true); + warmBothTables(); + if (dropPrimaryTable) { + dropTable("cache_replay.target", false); + } else { + target.dropConstraint("cache_pk", true); + } + Assertions.assertFalse(control.getConstraintsMapUnsafe().containsKey("cache_fk")); + Assertions.assertEquals(0, Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().estimatedSize()); + Assertions.assertTrue(aggregates.invalidated.containsAll(ImmutableList.of(target.getId(), control.getId()))); + } + + @Test + void replayLegacyCreateTableInvalidatesAllTableCaches() throws Exception { + warmBothTables(); + JsonObject legacy = GsonUtils.GSON.toJsonTree( + new CreateTableInfo(database.getFullName(), database.getId(), target)).getAsJsonObject(); + for (String field : ImmutableList.of("ctl", "tbl", "dbId")) { + legacy.remove(field); + } + CreateTableInfo info = GsonUtils.GSON.fromJson(legacy, CreateTableInfo.class); + Assertions.assertNull(info.getCtlName()); + Assertions.assertNull(info.getTblName()); + Assertions.assertEquals(-1L, info.getDbId()); + database.unregisterTable(target.getId()); + + replay(OperationType.OP_CREATE_TABLE, info); + + Assertions.assertEquals(target.getId(), database.getTableOrMetaException("target").getId()); + Assertions.assertNotSame(target, database.getTableNullable("target")); + assertTargetInvalidated(); + } + + @Test + void replayRenameColumnInvalidatesAllTableCaches() throws Exception { + warmBothTables(); + long dataVersion = target.getVisibleVersion(); + TableRenameColumnInfo info = new TableRenameColumnInfo(database.getId(), target.getId(), "v", "new_v", + ImmutableMap.of(target.getBaseIndexId(), target.getBaseSchemaVersion() + 1)); + replay(OperationType.OP_RENAME_COLUMN, info); + Assertions.assertNull(target.getColumn("v")); + Assertions.assertNotNull(target.getColumn("new_v")); + Assertions.assertEquals(dataVersion, target.getVisibleVersion()); + assertTargetInvalidated(); + } + + @Test + void replayRenameTableRemovesSortedEntryUnderOldName() throws Exception { + warmBothTables(); + List namesAtNotification = new ArrayList<>(); + aggregates.onInvalidate = () -> namesAtNotification.add(target.getName()); + replay(OperationType.OP_RENAME_TABLE, + TableInfo.createForTableRename(database.getId(), target.getId(), "target", "renamed")); + Assertions.assertEquals("renamed", target.getName()); + Assertions.assertEquals(ImmutableList.of("renamed"), namesAtNotification); + assertTargetInvalidated(); + getSQLPlanOrErrorMsg("SELECT id FROM cache_replay.renamed WHERE id=1"); + Assertions.assertEquals(2, Env.getCurrentEnv().getSortedPartitionsCacheManager() + .getPartitionCaches().estimatedSize()); + } + + @ParameterizedTest + @CsvSource({"false", "true"}) + void replayReplaceTableInvalidatesBothIdentities(boolean swap) throws Exception { + warmBothTables(); + List identitiesAtNotification = new ArrayList<>(); + aggregates.onInvalidate = () -> identitiesAtNotification.add(database.getTableNullable("target").getId()); + replay(OperationType.OP_REPLACE_TABLE, new ReplaceTableOperationLog(database.getId(), + target.getId(), "target", control.getId(), "control", swap, false)); + Assertions.assertSame(control, database.getTableNullable("target")); + Assertions.assertFalse(identitiesAtNotification.isEmpty()); + Assertions.assertTrue(identitiesAtNotification.stream().allMatch(id -> id == control.getId())); + if (swap) { + Assertions.assertSame(target, database.getTableNullable("control")); + } else { + Assertions.assertNull(database.getTableNullable("control")); + } + Assertions.assertEquals(0, Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().estimatedSize()); + Assertions.assertEquals(0, Env.getCurrentEnv().getSortedPartitionsCacheManager() + .getPartitionCaches().estimatedSize()); + Assertions.assertTrue(aggregates.invalidated.containsAll(ImmutableList.of(target.getId(), control.getId()))); + } + + @Test + void replayViewDefinitionInvalidatesQueriesUsingTheView() throws Exception { + executeNereidsSql("DROP VIEW IF EXISTS cache_replay.cached_view"); + createView("CREATE VIEW cache_replay.cached_view AS SELECT id FROM cache_replay.target"); + View view = (View) database.getTableOrMetaException("cached_view"); + SqlCacheContext context = warmSql("view", target); + context.addUsedView(view, view.getInlineViewDef()); + warmSql("control", control); + replay(OperationType.OP_MODIFY_VIEW_DEF, new AlterViewInfo(database.getId(), view.getId(), + "SELECT v FROM cache_replay.target", view.getFullSchema(), ImmutableMap.of(), "")); + Assertions.assertNull(Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().getIfPresent("view")); + Assertions.assertNotNull(Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().getIfPresent("control")); + } + + @Test + void visibleReplayDoesNotScanCachesAndValidatesDependenciesOnLookup() throws Exception { + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + String first = "SELECT id FROM target PARTITION(p1)"; + String second = "SELECT id FROM target PARTITION(p2)"; + String whole = "SELECT id FROM target"; + String other = "SELECT id FROM control PARTITION(p1)"; + prepareCachedQuery(first, target); + publishCachedQuery(false, first); + prepareCachedQuery(second, target, "p2"); + publishCachedQuery(false, second); + prepareCachedQuery(whole, target, null); + publishCachedQuery(false, whole); + prepareCachedQuery(other, control); + publishCachedQuery(false, other); + Assertions.assertEquals(4, manager.getSqlCaches().estimatedSize()); + manager.getSqlCaches().asMap().replaceAll((key, cached) -> Mockito.spy(cached)); + List cachedContexts = new ArrayList<>(manager.getSqlCaches().asMap().values()); + + TransactionState transaction = committedTransaction(target, "p2"); + transaction.setTransactionStatus(TransactionStatus.VISIBLE); + replay(OperationType.OP_UPSERT_TRANSACTION_STATE, transaction); + + for (SqlCacheContext cached : cachedContexts) { + Mockito.verify(cached, Mockito.never()).getUsedTables(); + } + Assertions.assertEquals(4, manager.getSqlCaches().estimatedSize(), + "VISIBLE must publish versions without walking or evicting cached queries under table write locks"); + assertCacheHit(first, false); + assertCacheHit(second, false); + assertCacheHit(whole, false); + assertCacheHit(other, true); + Assertions.assertEquals(1, manager.getSqlCaches().estimatedSize()); + Assertions.assertTrue(aggregates.invalidated.isEmpty()); + } + + @ParameterizedTest + @CsvSource({"false", "true"}) + void transactionReplayChecksDataWhenVisibleButNotWhenCommitted(boolean partitioned) throws Exception { + OlapTable table = partitioned ? target : (OlapTable) database.getTableOrMetaException("plain"); + String partitionName = partitioned ? "p1" : table.getName(); + String query = "SELECT id FROM " + table.getName(); + prepareCachedQuery(query, table); + publishCachedQuery(false, query); + long oldVersion = table.getVisibleVersion(); + TransactionState transaction = committedTransaction(table, partitionName); + replay(OperationType.OP_UPSERT_TRANSACTION_STATE, transaction); + Assertions.assertEquals(oldVersion, table.getVisibleVersion()); + assertCacheHit(query, true); + + transaction.setTransactionStatus(TransactionStatus.VISIBLE); + replay(OperationType.OP_UPSERT_TRANSACTION_STATE, transaction); + Assertions.assertEquals(oldVersion + 1, table.getVisibleVersion()); + Assertions.assertEquals(1, Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().estimatedSize()); + assertCacheHit(query, false); + Assertions.assertEquals(0, Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().estimatedSize()); + Assertions.assertTrue(aggregates.invalidated.isEmpty()); + } + + private TransactionState committedTransaction(OlapTable table, String partitionName) throws Exception { + long now = System.currentTimeMillis(); + TransactionState transaction = new TransactionState(database.getId(), ImmutableList.of(table.getId()), + Env.getCurrentEnv().getNextId(), "cache_replay_visible", null, + TransactionState.LoadJobSourceType.FRONTEND, + new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.FE, 0, "127.0.0.1", now), + -1, 10000); + TableCommitInfo commit = new TableCommitInfo(table.getId()); + commit.setVersion(table.getVisibleVersion() + 1); + commit.setVersionTime(now); + Partition partition = table.getPartition(partitionName); + commit.addPartitionCommitInfo(new PartitionCommitInfo(partition.getId(), "", + partition.getVisibleVersion() + 1, now, false)); + transaction.putIdToTableCommitInfo(table.getId(), commit); + transaction.setCommitTime(now); + transaction.setFinishTime(now); + transaction.setTransactionStatus(TransactionStatus.COMMITTED); + return transaction; + } + + private void assertCacheHit(String query, boolean expected) { + try (StatementContext lookup = new StatementContext(connectContext, new OriginStatement(query, 0))) { + connectContext.setStatementContext(lookup); + Assertions.assertEquals(expected, + Env.getCurrentEnv().getSqlCacheManager().tryParseSql(connectContext, query).isPresent(), query); + } + } + + @Test + void nonPartitionedCacheChecksSchemaAndChangesBackendKey() throws Exception { + OlapTable table = (OlapTable) database.getTableOrMetaException("plain"); + String query = "SELECT * FROM plain"; + SqlCacheContext cached = prepareCachedQuery(query, table); + Env env = Env.getCurrentEnv(); + env.getSqlCacheManager().tryAddFeSqlCache(connectContext, query); + String oldKey = cached.getUsedTables().values().iterator().next().getCacheKey(); + try (StatementContext lookup = new StatementContext(connectContext, new OriginStatement(query, 0))) { + connectContext.setStatementContext(lookup); + Assertions.assertTrue(env.getSqlCacheManager().tryParseSql(connectContext, query).isPresent()); + } + // Model a plan published after the eager invalidation has already run. + table.getIndexMetaByIndexId(table.getBaseIndexId()).setSchemaVersion(table.getBaseSchemaVersion() + 1); + SqlCacheContext current = new SqlCacheContext(UserIdentity.ROOT); + current.addUsedTable(table); + Assertions.assertNotEquals(oldKey, current.getUsedTables().values().iterator().next().getCacheKey()); + try (StatementContext lookup = new StatementContext(connectContext, new OriginStatement(query, 0))) { + connectContext.setStatementContext(lookup); + Assertions.assertFalse(env.getSqlCacheManager().tryParseSql(connectContext, query).isPresent()); + } + Assertions.assertEquals(0, env.getSqlCacheManager().getSqlCaches().estimatedSize()); + } + + @ParameterizedTest + @CsvSource({"false,false", "false,true", "true,false", "true,true"}) + void metadataReplayRejectsInFlightPublication(boolean rename, boolean backend) throws Exception { + target.getPartition("p1").updateVisibleVersionAndTime(3L, 123456L); + String query = "SELECT id FROM target PARTITION(p1)"; + SqlCacheContext cached = prepareCachedQuery(query, target); + StatementContext statement = connectContext.getStatementContext(); + SqlCacheContext unrelated = prepareCachedQuery("SELECT id FROM control", control); + connectContext.setStatementContext(statement); + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + long tableVersion = target.getVisibleVersion(); + int schemaVersion = target.getBaseSchemaVersion(); + + replayPartitionMetadataChange(rename); + + Assertions.assertEquals(tableVersion, target.getVisibleVersion()); + Assertions.assertEquals(schemaVersion, target.getBaseSchemaVersion()); + publishCachedQuery(backend, query); + Assertions.assertEquals(0, manager.getSqlCaches().estimatedSize(), + "an executing query must not republish its old metadata dependencies"); + Assertions.assertTrue(cached.supportSqlCache()); + Assertions.assertFalse(cached.metadataVersionsMatch()); + Assertions.assertTrue(unrelated.metadataVersionsMatch()); + } + + @ParameterizedTest + @CsvSource({"false", "true"}) + void metadataReplayRejectsContextPutAfterEviction(boolean rename) throws Exception { + target.getPartition("p1").updateVisibleVersionAndTime(3L, 123456L); + String query = "SELECT id FROM target PARTITION(p1)"; + SqlCacheContext cached = prepareCachedQuery(query, target); + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + manager.tryAddFeSqlCache(connectContext, query); + String key = manager.getSqlCaches().asMap().keySet().iterator().next(); + + replayPartitionMetadataChange(rename); + Assertions.assertEquals(0, manager.getSqlCaches().estimatedSize()); + // Model invalidation between the publisher's final check and its put. + manager.getSqlCaches().put(key, cached); + + try (StatementContext lookup = new StatementContext(connectContext, new OriginStatement(query, 0))) { + connectContext.setStatementContext(lookup); + Assertions.assertFalse(manager.tryParseSql(connectContext, query).isPresent(), + "a late put must not make an invalidated context usable again"); + } + Assertions.assertEquals(0, manager.getSqlCaches().estimatedSize()); + } + + @ParameterizedTest + @CsvSource({"false", "true"}) + void publicationValidatesDataWithoutAnInvalidationNotification(boolean backend) throws Exception { + target.getPartition("p1").updateVisibleVersionAndTime(3L, 123456L); + String query = "SELECT id FROM target PARTITION(p1)"; + SqlCacheContext cached = prepareCachedQuery(query, target); + target.getPartition("p1").updateVisibleVersionAndTime(4L, 123457L); + target.updateVisibleVersionAndTime(target.getVisibleVersion() + 1, 123457L); + Assertions.assertTrue(cached.metadataVersionsMatch()); + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + publishCachedQuery(backend, query); + Assertions.assertEquals(0, manager.getSqlCaches().estimatedSize()); + } + + @ParameterizedTest + @CsvSource({"false,false", "false,true", "true,false", "true,true"}) + void publicationPreservesBranchTableVersionValidation(boolean backend, boolean writeOtherPartition) + throws Exception { + String query = "SELECT id FROM target PARTITION(p1)"; + SqlCacheContext cached = prepareCachedQuery(query, target); + if (writeOtherPartition) { + target.writeLock(); + try { + Partition p2 = target.getPartition("p2"); + p2.updateVisibleVersionAndTime(p2.getVisibleVersion() + 1, 123457L); + target.updateVisibleVersionAndTime(target.getVisibleVersion() + 1, 123457L); + } finally { + target.writeUnlock(); + } + } + publishCachedQuery(backend, query); + Assertions.assertTrue(cached.metadataVersionsMatch()); + Assertions.assertEquals(writeOtherPartition ? 0 : 1, + Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().estimatedSize()); + } + + private void publishCachedQuery(boolean backend, String query) { + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + if (backend) { + CacheAnalyzer analyzer = Mockito.mock(CacheAnalyzer.class); + Mockito.when(analyzer.getCache()).thenReturn(Mockito.mock(SqlCache.class)); + Mockito.when(analyzer.getScanTables()).thenReturn(ImmutableList.of()); + manager.tryAddBeCache(connectContext, query, analyzer); + } else { + manager.tryAddFeSqlCache(connectContext, query); + } + } + + private void replayPartitionMetadataChange(boolean rename) throws Exception { + Partition partition = target.getPartition("p1"); + if (rename) { + replay(OperationType.OP_RENAME_PARTITION, TableInfo.createForPartitionRename( + database.getId(), target.getId(), partition.getId(), "p1", "renamed_partition")); + } else { + replay(OperationType.OP_SET_PARTITION_VERSION, new SetPartitionVersionOperationLog( + database.getFullName(), target.getName(), partition.getId(), 2L)); + } + } + + @ParameterizedTest + @CsvSource({"database", "catalog", "all"}) + void namespaceInvalidationReachesExecutingQueries(String scope) { + SqlCacheContext cached = prepareCachedQuery("SELECT id FROM target", target); + StatementContext statement = connectContext.getStatementContext(); + // A constant statement has no table dependencies. + StatementContext constantStatement = new StatementContext(connectContext, new OriginStatement("SELECT 1", 0)); + SqlCacheContext constant = constantStatement.getSqlCacheContext().get(); + connectContext.setStatementContext(statement); + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + // Replacing the result-cache container must preserve dependency revisions. + NereidsSqlCacheManager.updateConfig(); + switch (scope) { + case "database": + Env.getCurrentEnv().onDatabaseChanged(InternalCatalog.INTERNAL_CATALOG_NAME, "cache_replay"); + break; + case "catalog": + Env.getCurrentEnv().onCatalogChanged(InternalCatalog.INTERNAL_CATALOG_NAME); + break; + case "all": + manager.invalidateAll(); + break; + default: + throw new AssertionError(scope); + } + manager.tryAddFeSqlCache(connectContext, "SELECT id FROM target"); + Assertions.assertEquals(0, manager.getSqlCaches().estimatedSize()); + Assertions.assertFalse(cached.metadataVersionsMatch()); + Assertions.assertEquals(!scope.equals("all"), constant.metadataVersionsMatch()); + } + + private SqlCacheContext prepareCachedQuery(String query, OlapTable table) { + return prepareCachedQuery(query, table, "p1"); + } + + private SqlCacheContext prepareCachedQuery(String query, OlapTable table, String partitionName) { + StatementContext statement = new StatementContext(connectContext, new OriginStatement(query, 0)); + connectContext.setStatementContext(statement); + SqlCacheContext cached = statement.getSqlCacheContext().get(); + cached.addUsedTable(table); + if (table.isPartitionedTable() && partitionName != null) { + Partition partition = table.getPartition(partitionName); + ScanTable scan = new ScanTable(new FullTableName( + InternalCatalog.INTERNAL_CATALOG_NAME, database.getFullName(), table.getName())); + scan.addScanPartition(partition.getId()); + cached.addScanTable(scan, table); + } + cached.setPhysicalPlan("cached plan"); + cached.setColLabels(ImmutableList.of("id", "v")); + cached.setFieldInfos(ImmutableList.of()); + cached.setResultExprs(ImmutableList.of()); + cached.setQueryId(new TUniqueId(1, 1)); + cached.setResultSetInFe(new CommonResultSet( + new CommonResultSet.CommonResultSetMetaData(ImmutableList.of()), ImmutableList.of())); + connectContext.setQueryId(new TUniqueId(1, 1)); + return cached; + } + + @Test + void databaseInvalidationPreservesOtherNamespaces() { + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + warmSql("target", target); + SqlCacheContext other = new SqlCacheContext(UserIdentity.ROOT); + manager.getSqlCaches().put("constant", other); + Env.getCurrentEnv().onDatabaseChanged(InternalCatalog.INTERNAL_CATALOG_NAME, "cache_replay"); + Assertions.assertNull(manager.getSqlCaches().getIfPresent("target")); + Assertions.assertNotNull(manager.getSqlCaches().getIfPresent("constant")); + } + + @Test + void invalidationFailuresAreIsolatedPerCache() { + NereidsSqlCacheManager sql = Mockito.mock(NereidsSqlCacheManager.class); + NereidsSortedPartitionsCacheManager sorted = Mockito.mock(NereidsSortedPartitionsCacheManager.class); + MTMVService mtmv = Mockito.mock(MTMVService.class); + MTMVRelationManager relations = Mockito.mock(MTMVRelationManager.class); + Env env = Mockito.mock(Env.class, Mockito.CALLS_REAL_METHODS); + Mockito.when(env.getMtmvService()).thenReturn(mtmv); + Mockito.when(mtmv.getRelationManager()).thenReturn(relations); + Mockito.when(env.getSqlCacheManager()).thenReturn(sql); + Mockito.when(env.getSortedPartitionsCacheManager()).thenReturn(sorted); + Mockito.doThrow(new IllegalStateException("SQL invalidation failed")) + .when(sql).invalidateAboutTable(Mockito.eq(target.getId()), Mockito.any(FullTableName.class)); + Mockito.doThrow(new IllegalStateException("partition invalidation failed")) + .when(sorted).invalidateTable(target.getId()); + Mockito.doThrow(new AssertionError("MV invalidation failed")).when(relations) + .invalidateRewriteCaches(InternalCatalog.INTERNAL_CATALOG_NAME, "cache_replay", "target"); + Assertions.assertDoesNotThrow(() -> env.onTableChanged(target)); + Mockito.verify(sorted).invalidateTable(target.getId()); + Mockito.verify(relations).invalidateRewriteCaches( + InternalCatalog.INTERNAL_CATALOG_NAME, "cache_replay", "target"); + Assertions.assertTrue(aggregates.invalidated.contains(target.getId())); + } + + private SqlCacheContext warmSql(String key, OlapTable table) { + SqlCacheContext context = new SqlCacheContext(UserIdentity.ROOT, + Env.getCurrentEnv().getSqlCacheManager().newMetadataSnapshot()); + context.addUsedTable(table); + Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().put(key, context); + return context; + } + + private void warmBothTables() throws Exception { + getSQLPlanOrErrorMsg("SELECT id FROM cache_replay.target WHERE id=1"); + getSQLPlanOrErrorMsg("SELECT id FROM cache_replay.control WHERE id=1"); + warmSql("target", target); + warmSql("control", control); + Assertions.assertEquals(2, Env.getCurrentEnv().getSortedPartitionsCacheManager() + .getPartitionCaches().estimatedSize()); + } + + private void assertTargetInvalidated() { + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + Assertions.assertNull(manager.getSqlCaches().getIfPresent("target")); + Assertions.assertNotNull(manager.getSqlCaches().getIfPresent("control")); + Assertions.assertEquals(1, Env.getCurrentEnv().getSortedPartitionsCacheManager() + .getPartitionCaches().estimatedSize()); + Assertions.assertTrue(aggregates.invalidated.contains(target.getId())); + Assertions.assertFalse(aggregates.invalidated.contains(control.getId())); + } + + private void replay(short operation, Writable payload) throws Exception { + JournalEntity written = new JournalEntity(); + written.setOpCode(operation); + written.setData(payload); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + written.write(new DataOutputStream(bytes)); + JournalEntity read = new JournalEntity(); + read.readFields(new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))); + EditLog.loadJournal(Env.getCurrentEnv(), 1L, read); + } + + private static class RecordingSimpleAggCache extends SimpleAggCacheMgr { + private final Set invalidated = new HashSet<>(); + private Runnable onInvalidate = () -> { }; + + @Override + public void invalidateTable(long tableId) { + invalidated.add(tableId); + onInvalidate.run(); + } + } +} diff --git a/regression-test/data/nereids_p0/cache/sql_cache_metadata_replay.out b/regression-test/data/nereids_p0/cache/sql_cache_metadata_replay.out new file mode 100644 index 00000000000000..7bdf7d92f875ed --- /dev/null +++ b/regression-test/data/nereids_p0/cache/sql_cache_metadata_replay.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_drop -- +1 11 99 + +-- !after_drop -- +1 11 + +-- !before_rename -- +11 + +-- !after_rename -- +1 11 + +-- !after_insert -- +1 11 +2 22 + +-- !view_before -- +1 11 +2 22 + +-- !view_after -- +1 111 +2 122 + +-- !after_truncate -- + diff --git a/regression-test/suites/nereids_p0/cache/mv_with_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/mv_with_sql_cache.groovy index 6b9fe0d656deac..30545374f45e77 100644 --- a/regression-test/suites/nereids_p0/cache/mv_with_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/mv_with_sql_cache.groovy @@ -60,6 +60,15 @@ suite("mv_with_sql_cache") { } } + // Publishing a rollup changes the table metadata and invalidates the old cached plans. + def assertCacheInvalidatedAndRebuilt = { List queries -> + queries.each { assertNoCache(it) } + queries.each { query -> + sql(query) + assertHasCache(query) + } + } + def retryTestSqlCache = { int executeTimes, long intervalMillis, Closure closure -> Throwable throwable = null for (int i = 1; i <= executeTimes; ++i) { @@ -118,9 +127,7 @@ suite("mv_with_sql_cache") { create_sync_mv(dbName, tb_name1, mv_name2, mv_sql2) sleep(10 * 1000) - assertHasCache "select * from ${tb_name1}" - assertHasCache mv_sql1 - assertHasCache mv_sql2 + assertCacheInvalidatedAndRebuilt(["select * from ${tb_name1}", mv_sql1, mv_sql2]) assertNoCache mv_str sql mv_str @@ -181,9 +188,7 @@ suite("mv_with_sql_cache") { create_sync_mv(dbName, tb_name1, mv_name2, mv_sql2) sleep(10 * 1000) - assertHasCache "select * from ${tb_name1}" - assertHasCache mv_sql1 - assertHasCache mv_sql2 + assertCacheInvalidatedAndRebuilt(["select * from ${tb_name1}", mv_sql1, mv_sql2]) assertNoCache mv_str sql mv_str @@ -244,9 +249,7 @@ suite("mv_with_sql_cache") { create_sync_mv(dbName, tb_name1, mv_name2, mv_sql2) sleep(10 * 1000) - assertHasCache "select * from ${tb_name1}" - assertHasCache mv_sql1 - assertHasCache mv_sql2 + assertCacheInvalidatedAndRebuilt(["select * from ${tb_name1}", mv_sql1, mv_sql2]) assertNoCache mv_str sql mv_str diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy index ae4437075ec158..5f3a25756a1b2e 100644 --- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy @@ -103,7 +103,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select * from test_use_plan_cache" // create sql cache - sql "select * from test_use_plan_cache" + retryUntilHasSqlCache "select * from test_use_plan_cache" // use sql cache assertHasCache "select * from test_use_plan_cache" @@ -122,7 +122,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache2" - sql "select * from test_use_plan_cache2" + retryUntilHasSqlCache "select * from test_use_plan_cache2" assertHasCache "select * from test_use_plan_cache2" // NOTE: in cloud mode, add empty partition can not use cache, because the table version already update, @@ -149,14 +149,14 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from ${tb_name}" - sql "select * from ${tb_name}" + retryUntilHasSqlCache "select * from ${tb_name}" assertHasCache "select * from ${tb_name}" // insert overwrite data can not use cache sql "INSERT OVERWRITE table ${tb_name} PARTITION(p5) VALUES (5, 6);" sleep(10 * 1000) assertNoCache "select * from ${tb_name}" - sql "select * from ${tb_name}" + retryUntilHasSqlCache "select * from ${tb_name}" assertHasCache "select * from ${tb_name}" sql "alter table ${tb_name} add partition p6 values[('6'),('7'))" @@ -180,7 +180,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache3" - sql "select * from test_use_plan_cache3" + retryUntilHasSqlCache "select * from test_use_plan_cache3" assertHasCache "select * from test_use_plan_cache3" // drop partition can not use cache @@ -210,7 +210,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache4" - sql "select * from test_use_plan_cache4" + retryUntilHasSqlCache "select * from test_use_plan_cache4" assertHasCache "select * from test_use_plan_cache4" // replace partition can not use cache @@ -231,7 +231,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache5" - sql "select * from test_use_plan_cache5" + retryUntilHasSqlCache "select * from test_use_plan_cache5" assertHasCache "select * from test_use_plan_cache5" streamLoad { @@ -258,7 +258,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache6" - sql "select * from test_use_plan_cache6" + retryUntilHasSqlCache "select * from test_use_plan_cache6" assertHasCache "select * from test_use_plan_cache6" sql "update test_use_plan_cache6 set value=3 where id=1" @@ -280,7 +280,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache7" - sql "select * from test_use_plan_cache7" + retryUntilHasSqlCache "select * from test_use_plan_cache7" assertHasCache "select * from test_use_plan_cache7" sql "delete from test_use_plan_cache7 where id = 1" @@ -302,7 +302,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache8" - sql "select * from test_use_plan_cache8" + retryUntilHasSqlCache "select * from test_use_plan_cache8" assertHasCache "select * from test_use_plan_cache8" sql "drop table test_use_plan_cache8" @@ -330,7 +330,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache9_view" - sql "select * from test_use_plan_cache9_view" + retryUntilHasSqlCache "select * from test_use_plan_cache9_view" assertHasCache "select * from test_use_plan_cache9_view" // alter view should not use cache @@ -354,7 +354,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache10_view" - sql "select * from test_use_plan_cache10_view" + retryUntilHasSqlCache "select * from test_use_plan_cache10_view" assertHasCache "select * from test_use_plan_cache10_view" sql "drop view test_use_plan_cache10_view" @@ -381,7 +381,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache11_view" - sql "select * from test_use_plan_cache11_view" + retryUntilHasSqlCache "select * from test_use_plan_cache11_view" assertHasCache "select * from test_use_plan_cache11_view" sql "insert into test_use_plan_cache11 values(1, 3)" @@ -414,7 +414,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache12" - sql "select * from test_use_plan_cache12" + retryUntilHasSqlCache "select * from test_use_plan_cache12" assertHasCache "select * from test_use_plan_cache12" sql "sync" @@ -467,7 +467,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache13" - sql "select * from test_use_plan_cache13" + retryUntilHasSqlCache "select * from test_use_plan_cache13" assertHasCache "select * from test_use_plan_cache13" } }).get() @@ -534,7 +534,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache14" - sql "select * from test_use_plan_cache14" + retryUntilHasSqlCache "select * from test_use_plan_cache14" assertHasCache "select * from test_use_plan_cache14" } }).get() @@ -591,7 +591,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache15" - sql "select * from test_use_plan_cache15" + retryUntilHasSqlCache "select * from test_use_plan_cache15" assertHasCache "select * from test_use_plan_cache15" } }).get() @@ -635,7 +635,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select random() from test_use_plan_cache16" assertNoCache "select year(now()) from test_use_plan_cache16" - sql "select year(now()) from test_use_plan_cache16" + retryUntilHasSqlCache "select year(now()) from test_use_plan_cache16" assertHasCache "select year(now()) from test_use_plan_cache16" assertNoCache "select second(now()) from test_use_plan_cache16" @@ -660,7 +660,7 @@ suite("parse_sql_from_sql_cache") { sql "set @custom_variable=10" assertNoCache "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" // create sql cache - sql "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" + retryUntilHasSqlCache "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" // can use sql cache assertHasCache "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" @@ -668,6 +668,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" def result2 = sql "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" + retryUntilHasSqlCache "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" assertHasCache "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" assertTrue(result2.size() == 1 && result2[0][0].toString().toInteger() == 20) @@ -684,6 +685,7 @@ suite("parse_sql_from_sql_cache") { def functionResult = sql functionVariableSql assertTrue(functionResult.size() == 1 && functionResult[0][0].toString().toInteger() == 10) + retryUntilHasSqlCache functionVariableSql assertHasCache functionVariableSql sql "set @custom_variable_in_function=-20" @@ -691,6 +693,7 @@ suite("parse_sql_from_sql_cache") { functionResult = sql functionVariableSql assertTrue(functionResult.size() == 1 && functionResult[0][0].toString().toInteger() == 20) + retryUntilHasSqlCache functionVariableSql assertHasCache functionVariableSql // switch back to the original value and reuse its value-aware cache @@ -704,6 +707,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" def res = sql "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" assertTrue(res[0][0] == 1) + retryUntilHasSqlCache "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" assertHasCache "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" sql "set @custom_variable2=2" @@ -711,6 +715,7 @@ suite("parse_sql_from_sql_cache") { // should not invalidate cache with @custom_variable2=1 res = sql "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" assertTrue(res[0][0] == 2) + retryUntilHasSqlCache "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" assertHasCache "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" sql "set @custom_variable2=1" @@ -805,7 +810,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache18" - sql "select * from test_use_plan_cache18" + retryUntilHasSqlCache "select * from test_use_plan_cache18" assertHasCache "select * from test_use_plan_cache18" } @@ -850,6 +855,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select * from test_use_plan_cache19 order by 1, 2" def result2 = sql "select * from test_use_plan_cache19 order by 1, 2" assertTrue(result2.size() > 1) + retryUntilHasSqlCache "select * from test_use_plan_cache19 order by 1, 2" assertHasCache "select * from test_use_plan_cache19 order by 1, 2" sql "set dry_run_query=true" @@ -879,11 +885,12 @@ suite("parse_sql_from_sql_cache") { def result1 = sql "select * from (select $randomInt as id)a" assertTrue(result1.size() == 1) + retryUntilHasSqlCache "select * from (select $randomInt as id)a" assertHasCache "select * from (select $randomInt as id)a" def result2 = sql "select * from (select $randomInt as id)a" assertTrue(result2.size() == 1) - sql "select * from test_use_plan_cache20 limit 0" + retryUntilHasSqlCache "select * from test_use_plan_cache20 limit 0" assertHasCache "select * from test_use_plan_cache20 limit 0" def result4 = sql "select * from test_use_plan_cache20 limit 0" assertTrue(result4.isEmpty()) @@ -891,6 +898,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select * from test_use_plan_cache20 where id=999" def result5 = sql "select * from test_use_plan_cache20 where id=999" assertTrue(result5.isEmpty()) + retryUntilHasSqlCache "select * from test_use_plan_cache20 where id=999" assertHasCache "select * from test_use_plan_cache20 where id=999" def result6 = sql "select * from test_use_plan_cache20 where id=999" assertTrue(result6.isEmpty()) @@ -926,6 +934,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select * from test_use_plan_cache21" def result1 = sql "select * from test_use_plan_cache21" assertTrue(result1.size() == 2) + retryUntilHasSqlCache "select * from test_use_plan_cache21" assertHasCache "select * from test_use_plan_cache21" sql "truncate table test_use_plan_cache21 partition dt2" @@ -947,7 +956,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment2*/ * from test_use_plan_cache22 order by 1, 2" - sql "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment1*/ * from test_use_plan_cache22 order by 1, 2" + retryUntilHasSqlCache "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment1*/ * from test_use_plan_cache22 order by 1, 2" assertHasCache "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment2*/ * from test_use_plan_cache22 order by 1, 2" } @@ -965,7 +974,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" int randomInt = Math.random() * 2000000000 - sql "select ${randomInt} from test_use_plan_cache23" + retryUntilHasSqlCache "select ${randomInt} from test_use_plan_cache23" profile("sql_cache_23_${randomInt}") { run { sql "/* sql_cache_23_${randomInt} */ select ${randomInt} from test_use_plan_cache23" @@ -978,7 +987,7 @@ suite("parse_sql_from_sql_cache") { } randomInt = Math.random() * 2000000000 - sql "select * from (select $randomInt as id)a" + retryUntilHasSqlCache "select * from (select $randomInt as id)a" profile("sql_cache_23_${randomInt}_2") { run { sql "/* sql_cache_23_${randomInt}_2 */ select * from (select $randomInt as id)a" @@ -1086,6 +1095,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" assertTrue((sql "select * from test_use_plan_cache24").size() > 0) + retryUntilHasSqlCache "select * from test_use_plan_cache24" assertHasCache "select * from test_use_plan_cache24" connectToDoris(context.config.jdbcUser, context.config.jdbcPassword, context.jdbcUrl) { @@ -1114,7 +1124,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select count(1) from test_use_plan_cache25 group by id order by id" + retryUntilHasSqlCache "select count(1) from test_use_plan_cache25 group by id order by id" assertHasCache "select count(1) from test_use_plan_cache25 group by id order by id" sql "alter table test_use_plan_cache25 rename column id id2" @@ -1135,7 +1145,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select count(1) from test_use_plan_cache26 group by id order by id" + retryUntilHasSqlCache "select count(1) from test_use_plan_cache26 group by id order by id" assertHasCache "select count(1) from test_use_plan_cache26 group by id order by id" sql "alter table test_use_plan_cache26 rename test_use_plan_cache26_bak" @@ -1157,7 +1167,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select count(1) from test_use_plan_cache27 group by id order by id" + retryUntilHasSqlCache "select count(1) from test_use_plan_cache27 group by id order by id" assertHasCache "select count(1) from test_use_plan_cache27 group by id order by id" sql "alter table test_use_plan_cache27 replace with table test_use_plan_cache27_bak" @@ -1174,7 +1184,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select count(1) from test_use_plan_cache28 group by id order by id" + retryUntilHasSqlCache "select count(1) from test_use_plan_cache28 group by id order by id" assertHasCache "select count(1) from test_use_plan_cache28 group by id order by id" sql "truncate table test_use_plan_cache28" @@ -1191,7 +1201,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select * from test_use_plan_cache29" + retryUntilHasSqlCache "select * from test_use_plan_cache29" assertHasCache "select * from test_use_plan_cache29" sql "alter table test_use_plan_cache29 add column bbb int default '0'" @@ -1209,7 +1219,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select * from test_use_plan_cache30" + retryUntilHasSqlCache "select * from test_use_plan_cache30" assertHasCache "select * from test_use_plan_cache30" sql "alter table test_use_plan_cache30 drop column value" @@ -1226,7 +1236,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select * from test_use_plan_cache31" + retryUntilHasSqlCache "select * from test_use_plan_cache31" assertHasCache "select * from test_use_plan_cache31" sql "alter table test_use_plan_cache31 modify column value bigint" @@ -1245,7 +1255,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select * from test_use_plan_cache32_view" + retryUntilHasSqlCache "select * from test_use_plan_cache32_view" assertHasCache "select * from test_use_plan_cache32_view" sql "insert into test_use_plan_cache32 values (1, 1)" diff --git a/regression-test/suites/nereids_p0/cache/sql_cache_metadata_replay.groovy b/regression-test/suites/nereids_p0/cache/sql_cache_metadata_replay.groovy new file mode 100644 index 00000000000000..294bd78d1cf48d --- /dev/null +++ b/regression-test/suites/nereids_p0/cache/sql_cache_metadata_replay.groovy @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("sql_cache_metadata_replay") { + withGlobalLock("cache_last_version_interval_second") { + def follower = sql_return_maparray("SHOW FRONTENDS").find { + it.Alive.toString().equalsIgnoreCase("true") && it.Join.toString().equalsIgnoreCase("true") + && !it.IsMaster.toString().equalsIgnoreCase("true") + } + if (follower == null) { + logger.info("SQL cache metadata replay requires a follower or observer FE") + return + } + def database = (sql "SELECT DATABASE()")[0][0] + def url = "jdbc:mysql://${follower.Host}:${follower.QueryPort}/${database}?useLocalSessionState=true" + connect(context.config.jdbcUser, context.config.jdbcPassword, url) { + def originalInterval = (sql "ADMIN SHOW FRONTEND CONFIG LIKE 'cache_last_version_interval_second'")[0][1] + sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second'='0')" + try { + sql "SET enable_sql_cache=true" + sql "SET enable_strong_consistency_read=true" + sql "DROP VIEW IF EXISTS sql_cache_replay_view" + sql "DROP TABLE IF EXISTS sql_cache_replay_table" + sql """CREATE TABLE sql_cache_replay_table (id INT, value INT, added INT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ('replication_num'='1', 'light_schema_change'='false')""" + sql "INSERT INTO sql_cache_replay_table VALUES (1,11,99)" + order_qt_before_drop "SELECT * FROM sql_cache_replay_table ORDER BY id" + retryUntilHasSqlCache("SELECT * FROM sql_cache_replay_table ORDER BY id") + explain { + sql "PHYSICAL PLAN SELECT * FROM sql_cache_replay_table ORDER BY id" + contains "PhysicalSqlCache" + } + sql "ALTER TABLE sql_cache_replay_table DROP COLUMN added" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE TableName='sql_cache_replay_table' + ORDER BY JobId DESC LIMIT 1""" + time 120 + } + order_qt_after_drop "SELECT * FROM sql_cache_replay_table ORDER BY id" + sql "ALTER TABLE sql_cache_replay_table SET ('light_schema_change'='true')" + sql "SYNC" + order_qt_before_rename "SELECT value FROM sql_cache_replay_table ORDER BY id" + retryUntilHasSqlCache("SELECT value FROM sql_cache_replay_table ORDER BY id") + explain { + sql "PHYSICAL PLAN SELECT value FROM sql_cache_replay_table ORDER BY id" + contains "PhysicalSqlCache" + } + sql "ALTER TABLE sql_cache_replay_table RENAME COLUMN value renamed_value" + sql "SYNC" + test { + sql "SELECT value FROM sql_cache_replay_table ORDER BY id" + exception "Unknown column" + } + order_qt_after_rename "SELECT id,renamed_value FROM sql_cache_replay_table ORDER BY id" + sql "INSERT INTO sql_cache_replay_table VALUES (2,22)" + order_qt_after_insert "SELECT id,renamed_value FROM sql_cache_replay_table ORDER BY id" + sql "CREATE VIEW sql_cache_replay_view AS SELECT id,renamed_value FROM sql_cache_replay_table" + order_qt_view_before "SELECT * FROM sql_cache_replay_view ORDER BY id" + sql """ALTER VIEW sql_cache_replay_view AS + SELECT id,renamed_value+100 AS renamed_value FROM sql_cache_replay_table""" + sql "SYNC" + order_qt_view_after "SELECT * FROM sql_cache_replay_view ORDER BY id" + sql "TRUNCATE TABLE sql_cache_replay_table" + sql "SYNC" + order_qt_after_truncate "SELECT * FROM sql_cache_replay_view ORDER BY id" + } finally { + sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second'='${originalInterval}')" + } + } + } +}