From 4c2a2b3698216616bb1b934c190c6cf8b8ffd583 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 9 Nov 2026 13:41:54 +0800 Subject: [PATCH 1/4] [fix](cache) Invalidate query caches when replaying table changes ### What problem does this PR solve? Related PR: #63612 Problem Summary: Follower and observer FEs can reuse stale SQL results and metadata-derived caches after applying table changes that do not advance data visible versions. Backport the shared metadata-change notification, per-cache failure isolation, and common SQL cache admission/lookup validation to this release branch's native cache and constraint interfaces. Preserve table-wide data version checks and fence older asynchronous cache builds. ### Release note Fix stale cached query results on follower and observer FEs after table metadata changes. ### Check List (For Author) - Test: Unit Test: 232 focused cache/replay and affected existing FE tests; FE build and Checkstyle. Multi-FE/BE regression not run locally. - Behavior changed: Yes, reject cached results whose metadata dependencies changed. - Does this need documentation: No (cherry picked from commit f066153277d96aa4346a4749781da8a357d87b8e) --- .../java/org/apache/doris/alter/Alter.java | 11 +- .../org/apache/doris/alter/RollupJobV2.java | 1 + .../apache/doris/alter/SchemaChangeJobV2.java | 1 + .../org/apache/doris/backup/RestoreJob.java | 12 +- .../java/org/apache/doris/catalog/Env.java | 96 ++- .../org/apache/doris/catalog/TableIf.java | 9 + .../NereidsSortedPartitionsCacheManager.java | 13 + .../common/cache/NereidsSqlCacheManager.java | 146 +++-- .../cache/SqlCacheMetadataVersions.java | 101 +++ .../apache/doris/datasource/CatalogMgr.java | 1 + .../doris/datasource/ExternalCatalog.java | 8 +- .../datasource/ExternalMetaCacheMgr.java | 198 +++--- .../doris/datasource/InternalCatalog.java | 7 +- .../org/apache/doris/mtmv/BaseTableInfo.java | 11 + .../doris/mtmv/MTMVRelationManager.java | 58 +- .../apache/doris/nereids/SqlCacheContext.java | 36 +- .../doris/nereids/StatementContext.java | 8 +- .../rules/analysis/CollectRelation.java | 36 +- .../org/apache/doris/persist/EditLog.java | 41 ++ .../apache/doris/alter/RollupJobV2Test.java | 8 + .../MTMVRewriteCacheInvalidationTest.java | 88 +++ .../cache/SqlCacheMetadataVersionsTest.java | 99 +++ .../ExternalMetaCacheRouteResolverTest.java | 2 + .../ExternalQueryCacheInvalidationTest.java | 153 +++++ ...reamingInsertJobOffsetPersistenceTest.java | 2 + .../StreamingInsertTaskAuditTest.java | 2 + .../MTMVRelationCacheInvalidationTest.java | 189 ++++++ .../doris/nereids/StatementContextTest.java | 45 ++ .../analysis/CollectRelationMTMVLockTest.java | 313 +++++++++ .../analysis/LocalTvfSinkDisabledTest.java | 2 + .../doris/qe/AuditLogWorkloadGroupTest.java | 2 + .../MysqlPreparedStatementForwardingTest.java | 2 + .../doris/qe/TableCacheInvalidationTest.java | 617 ++++++++++++++++++ .../cache/sql_cache_metadata_replay.out | 27 + .../cache/parse_sql_from_sql_cache.groovy | 74 ++- .../cache/sql_cache_metadata_replay.groovy | 86 +++ 36 files changed, 2299 insertions(+), 206 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/cache/SqlCacheMetadataVersions.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/MTMVRewriteCacheInvalidationTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/common/cache/SqlCacheMetadataVersionsTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalQueryCacheInvalidationTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationCacheInvalidationTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CollectRelationMTMVLockTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/TableCacheInvalidationTest.java create mode 100644 regression-test/data/nereids_p0/cache/sql_cache_metadata_replay.out create mode 100644 regression-test/suites/nereids_p0/cache/sql_cache_metadata_replay.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 181f84a8cdba0c..6d391513ce9e42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -72,7 +72,6 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.UserException; -import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.MetaLockUtils; import org.apache.doris.common.util.PropertyAnalyzer; @@ -364,8 +363,7 @@ private boolean processAlterOlapTableInternal(List alterClauses, Ol olapTable.writeLock(); try { - NereidsSqlCacheManager sqlCacheManager = Env.getCurrentEnv().getSqlCacheManager(); - sqlCacheManager.invalidateAboutTable(olapTable); + Env.getCurrentEnv().onTableChanged(olapTable); } finally { olapTable.writeUnlock(); } @@ -840,8 +838,11 @@ private void replaceTableInternal(Database db, OlapTable origTable, OlapTable ne // rename origin table name to new table name and add it to database origTable.checkAndSetName(newTblName, false); db.registerTable(origTable); - } else { - + } + // Advance revisions after the name mapping changes, retaining both original dependency identities. + Env.getCurrentEnv().onTableChanged(origTable, oldTblName); + Env.getCurrentEnv().onTableChanged(newTbl, newTblName); + if (!swapTable) { // not swap, the origin table is not used anymore, need to drop all its tablets. // put original table to recycle bin. if (isForce) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index 5012eacab13fa3..353187b85bf9a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -647,6 +647,7 @@ private void onFinished(OlapTable tbl) { tbl.getIndexIdToSchema(true)); } tbl.rebuildFullSchema(); + Env.getCurrentEnv().onTableChanged(tbl); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 57b24fae28f624..b8306605cc5021 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -839,6 +839,7 @@ private void onFinished(OlapTable tbl) { if (storageFormat == TStorageFormat.V2) { tbl.setStorageFormat(storageFormat); } + Env.getCurrentEnv().onTableChanged(tbl); } /* diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 7c0f52b93edf59..c55ac83ee202e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -2233,10 +2233,7 @@ protected Status allTabletCommitted(boolean isReplay) { } protected void updateOlapTablesVersion(Database db, boolean isReplay) { - if (Env.getCurrentEnv().invalidCacheForCloud()) { - return; - } - + boolean cloudCacheInvalidated = Env.getCurrentEnv().invalidCacheForCloud(); for (String tableName : jobInfo.backupOlapTableObjects.keySet()) { Table tbl = db.getTableNullable(jobInfo.getAliasByOriginNameIfSet(tableName)); if (tbl == null) { @@ -2252,8 +2249,11 @@ protected void updateOlapTablesVersion(Database db, boolean isReplay) { continue; } try { - long nextVersion = olapTbl.getNextVersion(); - olapTbl.updateVisibleVersionAndTime(nextVersion, System.currentTimeMillis()); + if (!cloudCacheInvalidated) { + long nextVersion = olapTbl.getNextVersion(); + olapTbl.updateVisibleVersionAndTime(nextVersion, System.currentTimeMillis()); + } + env.onTableChanged(olapTbl); } finally { olapTbl.writeUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 2c6fab3e9a87c9..26aa27cbf4ee28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -171,9 +171,11 @@ import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.SqlCacheContext.FullTableName; import org.apache.doris.nereids.jobs.load.LabelProcessor; import org.apache.doris.nereids.lineage.LineageEventProcessor; import org.apache.doris.nereids.stats.HboPlanStatisticsManager; +import org.apache.doris.nereids.stats.SimpleAggCacheMgr; import org.apache.doris.nereids.trees.plans.commands.AdminSetFrontendConfigCommand; import org.apache.doris.nereids.trees.plans.commands.AdminSetPartitionVersionCommand; import org.apache.doris.nereids.trees.plans.commands.AdminSetReplicaStatusCommand; @@ -3436,8 +3438,8 @@ public boolean invalidCacheForCloud() { if (Config.isCloudMode()) { // the cloud mode will not update table version when use recover partition, // so we should invalidate all cache to avoid bugs - getSqlCacheManager().invalidateAll(); - getSortedPartitionsCacheManager().invalidateAll(); + invalidateCache("SQL", () -> getSqlCacheManager().invalidateAll()); + invalidateCache("sorted partitions", () -> getSortedPartitionsCacheManager().invalidateAll()); return true; } return false; @@ -5735,6 +5737,7 @@ public void replayRenameTable(TableInfo tableInfo) throws MetaNotFoundException db.unregisterTable(tableName); table.setName(newTableName); db.registerTable(table); + onTableChanged(table, tableName); LOG.info("replay rename table[{}] to {}", tableName, newTableName); } finally { table.writeUnlock(); @@ -6336,6 +6339,7 @@ public void updateBinlogConfig(Database db, OlapTable table, BinlogConfig newBin Preconditions.checkArgument(table.isWriteLockHeldByCurrentThread()); table.setBinlogConfig(newBinlogConfig); + onTableChanged(table); ModifyTablePropertyOperationLog info = new ModifyTablePropertyOperationLog(db.getId(), table.getId(), table.getName(), @@ -6390,6 +6394,7 @@ public void replayModifyTableProperty(short opCode, ModifyTablePropertyOperation default: break; } + onTableChanged(olapTable); } finally { olapTable.writeUnlock(); } @@ -6740,6 +6745,7 @@ public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundExcep ExternalCatalog ctl = (ExternalCatalog) catalogMgr.getCatalog(info.getCtl()); if (ctl != null) { ctl.replayTruncateTable(info); + onTableChanged(info.getCtl(), info.getDb(), info.getTable()); } } } @@ -7113,7 +7119,7 @@ public void setTableStatusInternal(String dbName, String tableName, OlapTableSta LOG.warn("ignore set same state {} for table {}. is replay: {}.", olapTable.getState(), tableName, isReplay); } - Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(olapTable); + onTableChanged(olapTable); } finally { olapTable.writeUnlock(); } @@ -7221,7 +7227,7 @@ private void setReplicaVersionInternal(long tabletId, long backendId, Long versi LOG.info("set replica {} of tablet {} on backend {} as version {}, last success version {}, " + "last failed version {}, update time {}. is replay: {}", replica.getId(), tabletId, backendId, version, lastSuccessVersion, lastFailedVersion, updateTime, isReplay); - Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(table); + onTableChanged(table); } finally { table.writeUnlock(); } @@ -7302,7 +7308,7 @@ public int setPartitionVersionInternal(String database, String table, long parti + " {}.", partitionId, oldVersion, visibleVersion, database, table, isReplay); } - Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(olapTable); + onTableChanged(olapTable); } finally { olapTable.writeUnlock(); } @@ -7609,6 +7615,86 @@ public NereidsSortedPartitionsCacheManager getSortedPartitionsCacheManager() { return sortedPartitionsCacheManager; } + /** Invalidate derived query caches after applying an existing table metadata journal. */ + public void onTableChanged(TableIf table) { + onTableChanged(table, table.getName()); + } + + /** Notify after mutation while retaining the original name of a renamed or replaced table. */ + public void onTableChanged(TableIf table, String originalName) { + DatabaseIf db = table.getDatabase(); + FullTableName name = db == null || db.getCatalog() == null ? null + : new FullTableName(db.getCatalog().getName(), db.getFullName(), originalName); + onTableChanged(table.getId(), name, table instanceof OlapTable); + } + + /** The table may have been removed by the operation whose journal is being replayed. */ + public void onTableChanged(long dbId, long tableId) { + Database db = getInternalCatalog().getDbNullable(dbId); + Table table = db == null ? null : db.getTableNullable(tableId); + if (table != null) { + onTableChanged(table); + } else { + onTableChanged(tableId, null, true); + } + } + + private void onTableChanged(long tableId, FullTableName name, boolean olapTable) { + invalidateCache("SQL", () -> getSqlCacheManager().invalidateAboutTable(tableId, name)); + invalidateCache("sorted partitions", () -> { + getSortedPartitionsCacheManager().invalidateTable(tableId); + if (name != null) { + getSortedPartitionsCacheManager().invalidateTable(name.catalog, name.db, name.table); + } + }); + if (name != null) { + invalidateCache("MTMV rewrite", () -> getMtmvService().getRelationManager() + .invalidateRewriteCaches(name.catalog, name.db, name.table)); + } + if (olapTable) { + invalidateCache("simple aggregates", () -> SimpleAggCacheMgr.internalInstance().invalidateTable(tableId)); + } + } + + /** Name-based external refresh logs can arrive before a local table object has been loaded. */ + public void onTableChanged(String catalog, String db, String table) { + if (Strings.isNullOrEmpty(catalog) || InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalog)) { + Database database = getInternalCatalog().getDbNullable(db); + Table current = database == null ? null : database.getTableNullable(table); + if (current != null) { + onTableChanged(current); + return; + } + catalog = InternalCatalog.INTERNAL_CATALOG_NAME; + } + onTableChanged(-1L, new FullTableName(catalog, db, table), false); + } + + public void onDatabaseChanged(String catalog, String db) { + invalidateCache("SQL", () -> getSqlCacheManager().invalidateAboutDatabase(catalog, db)); + invalidateCache("sorted partitions", () -> getSortedPartitionsCacheManager().invalidateDatabase(catalog, db)); + invalidateCache("MTMV rewrite", () -> getMtmvService().getRelationManager() + .invalidateRewriteCaches(catalog, db, null)); + } + + public void onCatalogChanged(String catalog) { + invalidateCache("SQL", () -> getSqlCacheManager().invalidateAboutCatalog(catalog)); + invalidateCache("sorted partitions", () -> getSortedPartitionsCacheManager().invalidateCatalog(catalog)); + invalidateCache("MTMV rewrite", () -> getMtmvService().getRelationManager() + .invalidateRewriteCaches(catalog, null, null)); + } + + private void invalidateCache(String cache, Runnable invalidation) { + if (isCheckpointThread()) { + return; + } + try { + invalidation.run(); + } catch (Throwable t) { + LOG.warn("Failed to invalidate {} cache after applying a metadata change", cache, t); + } + } + public SplitSourceManager getSplitSourceManager() { return splitSourceManager; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index ef9a823fd01e31..8f4dac01d43cd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -321,6 +321,7 @@ default void addUniqueConstraint(String name, ImmutableList columns, boo UniqueConstraint uniqueConstraint = new UniqueConstraint(name, ImmutableSet.copyOf(columns)); checkConstraintNotExistenceUnsafe(name, uniqueConstraint, constraintMap); constraintMap.put(name, uniqueConstraint); + notifyConstraintChanged(getConstraintRelatedTables(uniqueConstraint)); if (!replay) { Env.getCurrentEnv().getEditLog().logAddConstraint( new AlterConstraintLog(uniqueConstraint, this)); @@ -332,6 +333,7 @@ default void addPrimaryKeyConstraint(String name, ImmutableList columns, PrimaryKeyConstraint primaryKeyConstraint = new PrimaryKeyConstraint(name, ImmutableSet.copyOf(columns)); checkConstraintNotExistenceUnsafe(name, primaryKeyConstraint, constraintMap); constraintMap.put(name, primaryKeyConstraint); + notifyConstraintChanged(getConstraintRelatedTables(primaryKeyConstraint)); if (!replay) { Env.getCurrentEnv().getEditLog().logAddConstraint( new AlterConstraintLog(primaryKeyConstraint, this)); @@ -363,6 +365,7 @@ default void addForeignConstraint(String name, ImmutableList columns, referencedTable); primaryKeyConstraint.addForeignTable(this); constraintMap.put(name, foreignKeyConstraint); + notifyConstraintChanged(getConstraintRelatedTables(foreignKeyConstraint)); if (!replay) { Env.getCurrentEnv().getEditLog().logAddConstraint( new AlterConstraintLog(foreignKeyConstraint, this)); @@ -407,6 +410,7 @@ default void dropConstraintRefWithLock(Constraint constraint) { MetaLockUtils.writeLockTables(tables); try { dropConstraintRef(constraint); + notifyConstraintChanged(tables); } finally { MetaLockUtils.writeUnlockTables(tables); } @@ -441,6 +445,11 @@ default List getConstraintRelatedTables(Constraint constraint) { return tables; } + default void notifyConstraintChanged(List relatedTables) { + Env.getCurrentEnv().onTableChanged(this); + relatedTables.forEach(table -> Env.getCurrentEnv().onTableChanged(table)); + } + default void dropConstraint(String name, boolean replay) { Map constraintMap = getConstraintsMapUnsafe(); if (!constraintMap.containsKey(name)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java index 2d670aa011ead7..8d4ed545c60c0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java @@ -71,6 +71,19 @@ public void invalidateTable(String catalog, String db, String table) { partitionCaches.invalidate(key); } + /** Match the stable ID as well, including entries under a table's previous name. */ + public void invalidateTable(long tableId) { + partitionCaches.asMap().entrySet().removeIf(entry -> entry.getValue().tableId == tableId); + } + + public void invalidateDatabase(String catalog, String db) { + partitionCaches.asMap().keySet().removeIf(key -> key.catalog.equals(catalog) && key.db.equals(db)); + } + + public void invalidateCatalog(String catalog) { + partitionCaches.asMap().keySet().removeIf(key -> key.catalog.equals(catalog)); + } + public Optional> get( SupportBinarySearchFilteringPartitions table, CatalogRelation scan) { ConnectContext connectContext = ConnectContext.get(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java index 31dd8bb170566f..b704a4e5978637 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java @@ -93,6 +93,9 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.stream.Stream; /** * NereidsSqlCacheManager @@ -102,6 +105,7 @@ public class NereidsSqlCacheManager { // key: :: // value: SqlCacheContext private volatile Cache sqlCaches; + private final SqlCacheMetadataVersions metadataVersions = new SqlCacheMetadataVersions(); public NereidsSqlCacheManager() { sqlCaches = buildSqlCaches( @@ -110,13 +114,16 @@ public NereidsSqlCacheManager() { ); } + public SqlCacheMetadataVersions.Snapshot newMetadataSnapshot() { + return metadataVersions.snapshot(); + } + @VisibleForTesting public Cache getSqlCaches() { return sqlCaches; } public void invalidateAboutTable(TableIf tableIf) { - Set invalidateKeys = new LinkedHashSet<>(); FullTableName invalidateTableName = null; DatabaseIf database = tableIf.getDatabase(); if (database != null) { @@ -128,32 +135,50 @@ public void invalidateAboutTable(TableIf tableIf) { } } - for (Entry kv : sqlCaches.asMap().entrySet()) { - String key = kv.getKey(); - SqlCacheContext context = kv.getValue(); - for (Entry nameToVersion : context.getUsedTables().entrySet()) { - FullTableName tableName = nameToVersion.getKey().tableName; - TableVersion tableVersion = nameToVersion.getValue(); - if (tableVersion.id == tableIf.getId()) { - invalidateKeys.add(key); - break; - } - if (tableName.equals(invalidateTableName)) { - invalidateKeys.add(key); - break; - } - } - } + invalidateAboutTable(tableIf.getId(), invalidateTableName); + } - for (String invalidateKey : invalidateKeys) { - sqlCaches.invalidate(invalidateKey); - } + /** Also accepts the identity of a table that has already been dropped or renamed. */ + public void invalidateAboutTable(long tableId, FullTableName invalidateTableName) { + metadataVersions.invalidateTable(tableId, invalidateTableName); + invalidateEntries(context -> context.getUsedTables().entrySet().stream() + .anyMatch(entry -> entry.getValue().id == tableId + || entry.getKey().tableName.equals(invalidateTableName)) + || context.getUsedViews().containsKey(invalidateTableName)); } public void invalidateAll() { + metadataVersions.invalidateAll(); sqlCaches.invalidateAll(); } + public void invalidateAboutDatabase(String catalog, String db) { + invalidateAboutNamespace(catalog, db); + } + + public void invalidateAboutCatalog(String catalog) { + invalidateAboutNamespace(catalog, null); + } + + private void invalidateAboutNamespace(String catalog, String db) { + metadataVersions.invalidateNamespace(catalog, db); + invalidateEntries(context -> Stream.concat( + context.getUsedTables().keySet().stream().map(reference -> reference.tableName), + context.getUsedViews().keySet().stream()) + .anyMatch(name -> name.catalog.equals(catalog) && (db == null || name.db.equals(db)))); + } + + private void invalidateEntries(Predicate affected) { + Set invalidateKeys = new LinkedHashSet<>(); + for (Entry entry : sqlCaches.asMap().entrySet()) { + SqlCacheContext context = entry.getValue(); + if (affected.test(context)) { + invalidateKeys.add(entry.getKey()); + } + } + sqlCaches.invalidateAll(invalidateKeys); + } + public static synchronized void updateConfig() { Env currentEnv = Env.getCurrentEnv(); if (currentEnv == null) { @@ -215,7 +240,7 @@ public void tryAddFeSqlCache(ConnectContext connectContext, String sql) { if (sqlCaches.getIfPresent(key) == null && sqlCacheContext.getOrComputeCacheKeyMd5(sessionVariable) != null && sqlCacheContext.getResultSetInFe().isPresent()) { sqlCacheContext.setAffectQueryResultVariables(sessionVariable); - sqlCaches.put(key, sqlCacheContext); + publishIfDependenciesValid(connectContext, key, sqlCacheContext); } } @@ -265,7 +290,7 @@ public void tryAddBeCache(ConnectContext connectContext, String sql, CacheAnalyz return; } - sqlCaches.put(key, sqlCacheContext); + publishIfDependenciesValid(connectContext, key, sqlCacheContext); } } @@ -346,24 +371,9 @@ private Optional tryParseSql( if (privilegeChanged(connectContext, env, sqlCacheContext)) { return invalidateCache(key); } - IsChanged tableIsChanged = tablesOrDataChanged(env, sqlCacheContext, connectContext.getStatementContext()); - if (tableIsChanged.changed) { - if (tableIsChanged.invalidCache) { - return invalidateCache(key); - } else { - return Optional.empty(); - } - } - IsChanged viewIsChanged = viewsChanged(env, sqlCacheContext); - if (viewIsChanged.changed) { - if (viewIsChanged.invalidCache) { - return invalidateCache(key); - } else { - return Optional.empty(); - } - } - if (externalCatalogChanged(env, sqlCacheContext)) { - return invalidateCache(key); + IsChanged dependencies = dependenciesChanged(env, sqlCacheContext, connectContext.getStatementContext()); + if (dependencies.changed) { + return dependencies.invalidCache ? invalidateCache(key) : Optional.empty(); } LogicalEmptyRelation whateverPlan = new LogicalEmptyRelation(new RelationId(0), ImmutableList.of()); @@ -432,8 +442,46 @@ private Optional tryParseSql( } } + private void publishIfDependenciesValid(ConnectContext context, String key, SqlCacheContext candidate) { + // Validate against fresh MVCC snapshots instead of the executing statement's retained metadata. + StatementContext executing = context.getStatementContext(); + try (StatementContext validation = new StatementContext(context, null)) { + context.setStatementContext(validation); + boolean changed = dependenciesChanged(context.getEnv(), candidate, validation).changed; + if (!changed) { + sqlCaches.put(key, candidate); + } + } catch (Throwable t) { + LOG.warn("Skip publishing SQL cache for query {} because dependency validation failed", + candidate.getQueryId(), t); + } finally { + context.setStatementContext(executing); + } + } + + // DML publishes table/partition versions; validate them here on admission and lookup instead of scanning + // cached queries while transaction publication or replay holds table write locks. Metadata journals also + // advance the independent revision fence because their mutations need not change data versions. + private IsChanged dependenciesChanged(Env env, SqlCacheContext context, StatementContext statement) { + if (!context.metadataVersionsMatch()) { + return IsChanged.CHANGED_AND_INVALIDATE_CACHE; + } + IsChanged tables = tablesOrDataChanged(env, context, statement); + if (tables.changed) { + return tables; + } + IsChanged views = viewsChanged(env, context); + if (views.changed) { + return views; + } + if (externalCatalogChanged(env, context) || !context.metadataVersionsMatch()) { + return IsChanged.CHANGED_AND_INVALIDATE_CACHE; + } + return IsChanged.NOT_CHANGED; + } + private IsChanged tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext, StatementContext statement) { - if (sqlCacheContext.hasUnsupportedTables()) { + if (!sqlCacheContext.supportSqlCache()) { return IsChanged.CHANGED_AND_INVALIDATE_CACHE; } @@ -457,6 +505,9 @@ private IsChanged tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext, } if (tableIf instanceof OlapTable) { OlapTable olapTable = (OlapTable) tableIf; + if (olapSchemaChanged(olapTable, tableVersion)) { + return IsChanged.CHANGED_AND_INVALIDATE_CACHE; + } long currentTableVersion = 0L; try { currentTableVersion = olapTable.getVisibleVersion(); @@ -523,6 +574,19 @@ private IsChanged tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext, return IsChanged.NOT_CHANGED; } + private boolean olapSchemaChanged(OlapTable table, TableVersion version) { + if (!table.tryReadLock(1, TimeUnit.MINUTES)) { + return true; + } + try { + return version.schemaVersion >= 0 + && (version.baseIndexId != table.getBaseIndexId() + || version.schemaVersion != table.getBaseSchemaVersion()); + } finally { + table.readUnlock(); + } + } + private IsChanged viewsChanged(Env env, SqlCacheContext sqlCacheContext) { for (Entry cacheView : sqlCacheContext.getUsedViews().entrySet()) { TableIf currentView = findTableIf(env, cacheView.getKey()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/SqlCacheMetadataVersions.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/SqlCacheMetadataVersions.java new file mode 100644 index 00000000000000..d1b42c1db73443 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/SqlCacheMetadataVersions.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.cache; + +import org.apache.doris.nereids.SqlCacheContext.FullTableName; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** Process-local metadata revisions shared by SQL cache dependencies, independent of data versions. */ +public final class SqlCacheMetadataVersions { + private final AtomicLong changes = new AtomicLong(); + private final AtomicLong all = new AtomicLong(); + // A dependency snapshot retains its revision cells. Unused scopes can be collected without tombstones. + private final Cache tableIds = Caffeine.newBuilder().weakValues().build(); + private final Cache names = Caffeine.newBuilder().weakValues().build(); + + public Snapshot snapshot() { + return new Snapshot(); + } + + public void invalidateTable(long tableId, FullTableName name) { + long revision = changes.incrementAndGet(); + if (tableId != -1L) { + advance(tableIds.getIfPresent(tableId), revision); + } + if (name != null) { + advance(names.getIfPresent(name), revision); + } + } + + public void invalidateNamespace(String catalog, String database) { + long revision = changes.incrementAndGet(); + advance(names.getIfPresent(new FullTableName(catalog, database, null)), revision); + } + + public void invalidateAll() { + all.accumulateAndGet(changes.incrementAndGet(), Math::max); + } + + private static void advance(AtomicLong version, long revision) { + // Name-only/cold logs need not create revision cells when no query depends on that scope. + if (version != null) { + version.accumulateAndGet(revision, Math::max); + } + } + + /** One query's captured revisions; no query/context is registered with the revision owner. */ + public final class Snapshot { + private final long start = changes.get(); + private final long allVersion = all.get(); + private final Map dependencies = new LinkedHashMap<>(); + private boolean capturedBeforeChange = true; + + public void addTable(long tableId, FullTableName name) { + // A refresh may retire an object before binding records its first dependency. Do not attach + // a new revision to metadata the statement could have resolved before that refresh. + capturedBeforeChange &= start == changes.get(); + capture(tableIds.get(tableId, ignored -> new AtomicLong())); + capture(names.get(name, ignored -> new AtomicLong())); + capture(names.get(new FullTableName(name.catalog, name.db, null), ignored -> new AtomicLong())); + capture(names.get(new FullTableName(name.catalog, null, null), ignored -> new AtomicLong())); + capturedBeforeChange &= start == changes.get(); + } + + private void capture(AtomicLong version) { + dependencies.putIfAbsent(version, version.get()); + } + + public boolean isCurrent() { + if (!capturedBeforeChange || allVersion != all.get()) { + return false; + } + for (Map.Entry dependency : dependencies.entrySet()) { + if (dependency.getKey().get() != dependency.getValue()) { + return false; + } + } + return true; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 4481d0f7259f6c..0bb6d16853e0f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -191,6 +191,7 @@ private void cleanupRemovedCatalog(RemovedCatalog removedCatalog, boolean perman Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalog(removedCatalog.catalogId); } Env.getCurrentEnv().getQueryStats().clear(removedCatalog.catalogId); + Env.getCurrentEnv().onCatalogChanged(removedCatalog.catalogName); LOG.info("Removed catalog with id {}, name {}", removedCatalog.catalogId, removedCatalog.catalogName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 204574d359501a..c8f4c8d0f61739 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -761,7 +761,11 @@ public void resetToUninitialized(boolean invalidCache) { Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(id); } } finally { - endTableLookupRefresh(); + try { + Env.getCurrentEnv().onCatalogChanged(name); + } finally { + endTableLookupRefresh(); + } } } @@ -1771,7 +1775,7 @@ private void invalidatePropertySpecificCacheGroups(Map updatedPr ExternalMetaCacheMgr extMetaCacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); if (java.util.Objects.nonNull(schemaCacheTtl) || updatedProps.containsKey(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY)) { - extMetaCacheMgr.removeCatalog(id); + extMetaCacheMgr.removeCatalog(id, name); return; } for (String key : updatedProps.keySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 91266c74747e7f..a4d42930d47f25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -395,22 +395,40 @@ private Map runtimeEffectiveCacheProperties( return effective; } - public void invalidateCatalog(long catalogId) { - rowCountCache.invalidateCatalog(catalogId); + private void invalidateCatalogCaches(long catalogId, String catalogName, Runnable invalidation) { try { + invalidation.run(); + } finally { + try { + if (catalogName != null) { + Env.getCurrentEnv().onCatalogChanged(catalogName); + } else { + Env.getCurrentEnv().getSortedPartitionsCacheManager().invalidateAll(); + } + } finally { + rowCountCache.invalidateCatalog(catalogId); + } + } + } + + public void invalidateCatalog(long catalogId) { + CatalogIf owner = getCatalog(catalogId); + invalidateCatalogCaches(catalogId, owner == null ? null : owner.getName(), () -> { + rowCountCache.invalidateCatalog(catalogId); invalidateLanceTableAccess(catalogId); routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateCatalog", () -> cache.invalidateCatalogEntries(catalogId))); - } finally { - rowCountCache.invalidateCatalog(catalogId); - } + }); } public void invalidateCatalogByEngine(long catalogId, String engine) { - routeSpecifiedEngine(engine, cache -> safeInvalidate( - cache, catalogId, "invalidateCatalogByEngine", - () -> cache.invalidateCatalogEntries(catalogId))); + CatalogIf owner = getCatalog(catalogId); + invalidateCatalogCaches(catalogId, owner == null ? null : owner.getName(), () -> { + routeSpecifiedEngine(engine, cache -> safeInvalidate( + cache, catalogId, "invalidateCatalogByEngine", + () -> cache.invalidateCatalogEntries(catalogId))); + }); } /** @@ -429,15 +447,22 @@ public T withCatalogLifecycleLock(long catalogId, Supplier action) { } public void removeCatalog(long catalogId) { - Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); - lifecycleLock.lock(); - try { - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "removeCatalog", - () -> cache.invalidateCatalog(catalogId))); - } finally { - lifecycleLock.unlock(); - } + CatalogIf owner = getCatalog(catalogId); + removeCatalog(catalogId, owner == null ? null : owner.getName()); + } + + public void removeCatalog(long catalogId, String catalogName) { + invalidateCatalogCaches(catalogId, catalogName, () -> { + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "removeCatalog", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } + }); } /** @@ -454,16 +479,11 @@ public void removeCatalog(long catalogId) { * retained runtime, its group) and lets the next statement load a coherent generation. */ public void onCatalogOperationalContextChanged(long catalogId) { - try { - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "onCatalogOperationalContextChanged", - () -> cache.invalidateCatalogEntries(catalogId))); - } finally { - // The committed property change reset the catalog against a new target. Database objects - // already evicted from the smaller metadata cache never run removal callbacks, so publish - // one catalog-wide row-count fence to cover those cold scopes as well. - rowCountCache.invalidateCatalog(catalogId); - } + CatalogIf owner = getCatalog(catalogId); + invalidateCatalogCaches(catalogId, owner == null ? null : owner.getName(), () -> + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "onCatalogOperationalContextChanged", + () -> cache.invalidateCatalogEntries(catalogId)))); } public void removeCatalogPermanently(long catalogId) { @@ -495,36 +515,41 @@ public void removeCatalogPermanently(long catalogId) { /** Restore catalog properties and retire any group initialized from the rejected candidate atomically. */ public void rollbackCatalogProperties(ExternalCatalog catalog, Map oldProperties) { long catalogId = catalog.getId(); - Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); - lifecycleLock.lock(); - try { + CatalogIf owner = getCatalog(catalogId); + invalidateCatalogCaches(catalogId, owner == null ? null : owner.getName(), () -> { + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); try { - catalog.rollBackCatalogProps(oldProperties); - catalog.resetToUninitialized(false); + try { + catalog.rollBackCatalogProps(oldProperties); + catalog.resetToUninitialized(false); + } finally { + // Retire any group initialized from the rejected candidate even when the reset's + // own cleanup (JDBC/Trino/Lance close) throws; otherwise an already-initialized + // group keeps taking the fast path against the old target. + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "rollbackCatalogProperties", + () -> cache.invalidateCatalog(catalogId))); + } } finally { - // Retire any group initialized from the rejected candidate even when the reset's - // own cleanup (JDBC/Trino/Lance close) throws; otherwise an already-initialized - // group keeps taking the fast path against the old target. - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "rollbackCatalogProperties", - () -> cache.invalidateCatalog(catalogId))); + lifecycleLock.unlock(); } - } finally { - rowCountCache.invalidateCatalog(catalogId); - lifecycleLock.unlock(); - } + }); } public void removeCatalogByEngine(long catalogId, String engine) { - Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); - lifecycleLock.lock(); - try { - routeSpecifiedEngine(engine, cache -> safeInvalidate( - cache, catalogId, "removeCatalogByEngine", - () -> cache.invalidateCatalog(catalogId))); - } finally { - lifecycleLock.unlock(); - } + CatalogIf owner = getCatalog(catalogId); + invalidateCatalogCaches(catalogId, owner == null ? null : owner.getName(), () -> { + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeSpecifiedEngine(engine, cache -> safeInvalidate( + cache, catalogId, "removeCatalogByEngine", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } + }); } public void invalidateDb(long catalogId, String dbName) { @@ -550,6 +575,7 @@ void invalidateDb(long catalogId, long dbId, String dbName, boolean invalidateRo } private void invalidateDb(long catalogId, String dbName, OptionalLong dbId, boolean invalidateRowCountCache) { + CatalogIf owner = getCatalog(catalogId); if (invalidateRowCountCache) { invalidateDatabaseRowCount(catalogId, dbId); } @@ -557,8 +583,14 @@ private void invalidateDb(long catalogId, String dbName, OptionalLong dbId, bool routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateDb", () -> cache.invalidateDb(catalogId, dbName))); } finally { - if (invalidateRowCountCache) { - invalidateDatabaseRowCount(catalogId, dbId); + try { + if (owner != null) { + Env.getCurrentEnv().onDatabaseChanged(owner.getName(), dbName); + } + } finally { + if (invalidateRowCountCache) { + invalidateDatabaseRowCount(catalogId, dbId); + } } } } @@ -575,35 +607,30 @@ private void invalidateDatabaseRowCount(long catalogId, OptionalLong dbId) { public void invalidateDb(ExternalDatabase database) { long catalogId = database.getCatalog().getId(); - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "invalidateDb", () -> cache.invalidateDb(database))); + CatalogIf owner = getCatalog(catalogId); + try { + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "invalidateDb", () -> cache.invalidateDb(database))); + } finally { + if (owner != null) { + Env.getCurrentEnv().onDatabaseChanged(owner.getName(), database.getFullName()); + } + } } public void invalidateTable(long catalogId, String dbName, String tableName) { - Runnable rowCountFence = () -> rowCountCache.invalidateCatalog(catalogId); - try { - rowCountFence = resolveTableRowCountFence(catalogId, dbName, getCachedDb(catalogId, dbName), tableName); - rowCountFence.run(); + invalidateTableCaches(catalogId, dbName, tableName, () -> { invalidateLanceTableAccess(catalogId); routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateTable", () -> cache.invalidateTable(catalogId, dbName, tableName))); - } finally { - rowCountFence.run(); - } + }); } public void invalidateTableByEngine(long catalogId, String engine, String dbName, String tableName) { - Runnable rowCountFence = () -> rowCountCache.invalidateCatalog(catalogId); - try { - rowCountFence = resolveTableRowCountFence(catalogId, dbName, getCachedDb(catalogId, dbName), tableName); - rowCountFence.run(); - routeSpecifiedEngine(engine, cache -> safeInvalidate( - cache, catalogId, "invalidateTableByEngine", - () -> cache.invalidateTable(catalogId, dbName, tableName))); - } finally { - rowCountFence.run(); - } + invalidateTableCaches(catalogId, dbName, tableName, () -> routeSpecifiedEngine(engine, cache -> safeInvalidate( + cache, catalogId, "invalidateTableByEngine", + () -> cache.invalidateTable(catalogId, dbName, tableName)))); } private void invalidateLanceTableAccess(long catalogId) { @@ -617,15 +644,27 @@ private void invalidateLanceTableAccess(long catalogId) { public void invalidatePartitions(long catalogId, String dbName, String tableName, List partitions) { + invalidateTableCaches(catalogId, dbName, tableName, () -> + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "invalidatePartitions", + () -> cache.invalidatePartitions(catalogId, dbName, tableName, partitions)))); + } + + private void invalidateTableCaches(long catalogId, String dbName, String tableName, Runnable invalidation) { + CatalogIf owner = getCatalog(catalogId); Runnable rowCountFence = () -> rowCountCache.invalidateCatalog(catalogId); try { rowCountFence = resolveTableRowCountFence(catalogId, dbName, getCachedDb(catalogId, dbName), tableName); rowCountFence.run(); - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "invalidatePartitions", - () -> cache.invalidatePartitions(catalogId, dbName, tableName, partitions))); + invalidation.run(); } finally { - rowCountFence.run(); + try { + if (owner != null) { + Env.getCurrentEnv().onTableChanged(owner.getName(), dbName, tableName); + } + } finally { + rowCountFence.run(); + } } } @@ -841,7 +880,12 @@ public void invalidateTableCache(ExternalTable dorisTable) { routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateTable", () -> cache.invalidateTable(dorisTable))); } finally { - invalidateRowCountCache(dorisTable); + try { + Env.getCurrentEnv().onTableChanged(dorisTable.getCatalog().getName(), + dorisTable.getDb().getFullName(), dorisTable.getName()); + } finally { + invalidateRowCountCache(dorisTable); + } } if (LOG.isDebugEnabled()) { LOG.debug("invalid table cache for {}.{} in catalog {}", dorisTable.getRemoteDbName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index adda42ad5c1498..5e5d560dfe68df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -129,7 +129,6 @@ import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.nereids.stats.SimpleAggCacheMgr; import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand.IdType; import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; import org.apache.doris.persist.AlterDatabasePropertyInfo; @@ -1043,6 +1042,7 @@ public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop, Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(), db.getId(), table.getId()); table.removeTableIdentifierFromPrimaryTable(); db.unregisterTable(table.getId()); + Env.getCurrentEnv().onTableChanged(table); StopWatch watch = StopWatch.createStarted(); Env.getCurrentRecycleBin().recycleTable(db.getId(), table, isReplay, isForceDrop, recycleTime); watch.stop(); @@ -2084,7 +2084,7 @@ public void dropPartitionWithoutCheck(Database db, OlapTable olapTable, String p } } if (partition != null && !isTempPartition) { - SimpleAggCacheMgr.internalInstance().invalidateTable(olapTable.getId()); + Env.getCurrentEnv().onTableChanged(olapTable); } // Here, we only wait for the EventProcessor to finish processing the event, @@ -2127,7 +2127,7 @@ public void replayDropPartition(DropPartitionInfo info) throws MetaNotFoundExcep } } if (partition != null && !info.isTempPartition()) { - SimpleAggCacheMgr.internalInstance().invalidateTable(olapTable.getId()); + Env.getCurrentEnv().onTableChanged(olapTable); } olapTable.updateVisibleVersionAndTime(info.getVersion(), info.getVersionTime()); // Replay set new partition loaded flag to true for auto analyze. @@ -4011,6 +4011,7 @@ private List truncateTableInternal(OlapTable olapTable, List getMtmvsByBaseView(BaseTableInfo table) { return viewMTMVs.getOrDefault(table, ImmutableSet.of()); } + /** Only discard local rewrite plans; replay must not refresh an MV or load external metadata. */ + public void invalidateRewriteCaches(String catalog, String db, String table) { + Set affected = Sets.newHashSet(); + if (table != null) { + BaseTableInfo base = new BaseTableInfo(new TableNameInfo(catalog, db, table)); + affected.addAll(getMtmvsByBaseTable(base)); + affected.addAll(getMtmvsByBaseView(base)); + if (InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalog)) { + affected.add(base); + } + } else { + for (Map> relations : ImmutableList.of(tableMTMVs, viewMTMVs)) { + for (Map.Entry> entry : relations.entrySet()) { + BaseTableInfo base = entry.getKey(); + if (catalog.equals(base.getCtlName()) && (db == null || db.equals(base.getDbName()))) { + affected.addAll(entry.getValue()); + } + } + } + } + List targets = Lists.newArrayList(); + for (BaseTableInfo mv : affected) { + Database database = Env.getCurrentInternalCatalog().getDbNullable(mv.getDbName()); + Table current = database == null ? null : database.getTableNullable(mv.getTableName()); + if (current instanceof MTMV) { + targets.add((MTMV) current); + } + } + // The existing generation barrier also rejects plans whose construction was already in flight. + for (MTMV target : targets) { + try { + target.invalidateRewriteCache(); + } catch (Throwable failure) { + LOG.warn("Failed to invalidate rewrite cache for MV {} after table metadata changed: {}.{}.{}", + target.getId(), catalog, db, table, failure); + } + } + } + public Set getMtmvsByBaseTableOneLevelAndFromView(BaseTableInfo table) { return tableMTMVsOneLevelAndFromView.getOrDefault(table, ImmutableSet.of()); } @@ -150,24 +193,15 @@ private Set getMTMVInfos(List tableInfos) { } private Set getOrCreateMTMVs(BaseTableInfo baseTableInfo) { - if (!tableMTMVs.containsKey(baseTableInfo)) { - tableMTMVs.put(baseTableInfo, Sets.newConcurrentHashSet()); - } - return tableMTMVs.get(baseTableInfo); + return tableMTMVs.computeIfAbsent(baseTableInfo, ignored -> Sets.newConcurrentHashSet()); } private Set getOrCreateMTMVsView(BaseTableInfo baseTableInfo) { - if (!viewMTMVs.containsKey(baseTableInfo)) { - viewMTMVs.put(baseTableInfo, Sets.newConcurrentHashSet()); - } - return viewMTMVs.get(baseTableInfo); + return viewMTMVs.computeIfAbsent(baseTableInfo, ignored -> Sets.newConcurrentHashSet()); } private Set getOrCreateMTMVsOneLevelAndFromView(BaseTableInfo baseTableInfo) { - if (!tableMTMVsOneLevelAndFromView.containsKey(baseTableInfo)) { - tableMTMVsOneLevelAndFromView.put(baseTableInfo, Sets.newConcurrentHashSet()); - } - return tableMTMVsOneLevelAndFromView.get(baseTableInfo); + return tableMTMVsOneLevelAndFromView.computeIfAbsent(baseTableInfo, ignored -> Sets.newConcurrentHashSet()); } public void refreshMTMVCache(MTMVRelation relation, BaseTableInfo mtmvInfo) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java index 11e454b9442246..109c015d32503c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.Pair; +import org.apache.doris.common.cache.SqlCacheMetadataVersions; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.mvcc.MvccReadSelector; @@ -68,6 +69,7 @@ public class SqlCacheContext { private static final Logger LOG = LogManager.getLogger(SqlCacheContext.class); private final UserIdentity userIdentity; + private final SqlCacheMetadataVersions.Snapshot metadataVersions; private volatile TUniqueId queryId; // if contains udf/udaf/tableValuesFunction we can not process it and skip use sql cache private volatile boolean cannotProcessExpression; @@ -113,6 +115,12 @@ public class SqlCacheContext { /** SqlCacheContext */ public SqlCacheContext(UserIdentity userIdentity) { + this(userIdentity, new SqlCacheMetadataVersions().snapshot()); + } + + /** Capture metadata revisions from the SQL cache manager that will validate this statement. */ + public SqlCacheContext(UserIdentity userIdentity, SqlCacheMetadataVersions.Snapshot metadataVersions) { + this.metadataVersions = metadataVersions; if (userIdentity == null) { ConnectContext connectContext = ConnectContext.get(); if (connectContext != null) { @@ -125,6 +133,10 @@ public SqlCacheContext(UserIdentity userIdentity) { this.userIdentity = userIdentity; } + public boolean metadataVersionsMatch() { + return metadataVersions.isCurrent(); + } + public String getPhysicalPlan() { return physicalPlan; } @@ -200,6 +212,8 @@ public synchronized void addUsedTable(TableIf tableIf, Optional sn setCannotProcessExpression(true); return; } + metadataVersions.addTable(tableIf.getId(), + new FullTableName(catalog.getName(), database.getFullName(), tableIf.getName())); if (!catalog.isInternalCatalog() && !this.externalCatalogConfigs.containsKey(catalog.getName())) { this.externalCatalogConfigs.put(catalog.getName(), getCatalogConfigs(catalog)); } @@ -238,7 +252,9 @@ public synchronized void addUsedTable(TableIf tableIf, Optional sn new TableVersion( tableIf.getId(), version, - tableIf.getType(), sqlCacheVersion + tableIf.getType(), sqlCacheVersion, + tableIf instanceof OlapTable ? ((OlapTable) tableIf).getBaseIndexId() : -1L, + tableIf instanceof OlapTable ? ((OlapTable) tableIf).getBaseSchemaVersion() : -1 ) ); } @@ -258,6 +274,8 @@ public synchronized void addUsedView(TableIf tableIf, String ddlSql) { setCannotProcessExpression(true); return; } + metadataVersions.addTable(tableIf.getId(), + new FullTableName(catalog.getName(), database.getFullName(), tableIf.getName())); if (!catalog.isInternalCatalog() && !this.externalCatalogConfigs.containsKey(catalog.getName())) { this.externalCatalogConfigs.put(catalog.getName(), getCatalogConfigs(catalog)); } @@ -482,8 +500,7 @@ public synchronized PUniqueId doComputeCacheKeyMd5( .append("=") .append(tableVersion.id) .append("@") - .append(tableVersion.sqlCacheVersion == null ? Long.toString(tableVersion.version) - : "sql-cache:" + tableVersion.sqlCacheVersion.getVersion()); + .append(tableVersion.getCacheKey()); } for (Entry entry : usedViews.entrySet()) { cacheKey.append("|") @@ -635,10 +652,23 @@ public static class TableVersion { public final long version; public final TableType type; public final SqlCacheVersion sqlCacheVersion; + public final long baseIndexId; + public final int schemaVersion; + + public TableVersion(long id, long version, TableType type, SqlCacheVersion sqlCacheVersion) { + this(id, version, type, sqlCacheVersion, -1L, -1); + } public TableVersion(long id, long version, TableType type) { this(id, version, type, null); } + + /** Data-version and schema identity shared by FE validation and BE cache lookup. */ + public String getCacheKey() { + String dataVersion = sqlCacheVersion == null + ? Long.toString(version) : "sql-cache:" + sqlCacheVersion.getVersion(); + return schemaVersion < 0 ? dataVersion : dataVersion + ":schema:" + baseIndexId + ":" + schemaVersion; + } } /** CacheKeyType */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 7bca37d91b0790..767e360324f2ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -375,9 +375,9 @@ public StatementContext() { this(ConnectContext.get(), null, 0); } - // For StatementScopeIdGenerator + // For StatementScopeIdGenerator; its fallback only allocates IDs and must not retain a connection. public StatementContext(int initialId) { - this(ConnectContext.get(), null, initialId); + this(null, null, initialId); } public StatementContext(ConnectContext connectContext, OriginStatement originStatement) { @@ -395,8 +395,8 @@ private StatementContext(ConnectContext connectContext, OriginStatement originSt if (CacheAnalyzer.canUseSqlCache(connectContext.getSessionVariable())) { // cannot set the queryId here because the queryId for the current query is set // in the subsequent steps. - this.sqlCacheContext = new SqlCacheContext( - connectContext.getCurrentUserIdentity()); + this.sqlCacheContext = new SqlCacheContext(connectContext.getCurrentUserIdentity(), + Env.getCurrentEnv().getSqlCacheManager().newMetadataSnapshot()); if (originStatement != null) { this.sqlCacheContext.setOriginSql(originStatement.originStmt); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java index 1d29fe6ef171ef..5379ccd8dc1f91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java @@ -62,6 +62,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * Rule to bind relations in query plan. @@ -280,27 +281,30 @@ private void collectMTMVCandidates(TableIf table, CascadesContext cascadesContex for (MTMV mtmv : mtmvSet) { cascadesContext.getStatementContext().getCandidateMTMVs().add(mtmv); cascadesContext.getStatementContext().getMtmvRelatedTables().put(mtmv.getFullQualifiers(), mtmv); + List> baseTableQualifiers; mtmv.readMvLock(); try { - for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTables()) { - if (!baseTableInfo.isValid()) { - continue; - } - if (isDebugEnabled) { - LOG.debug("mtmv {} related base table include {}", new BaseTableInfo(mtmv), baseTableInfo); - } - try { - // Collect all base tables and lock them before querying - cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(), - TableFrom.MTMV, Optional.empty()); - } catch (AnalysisException exception) { - LOG.warn("mtmv related base table get err, related table is {}", - baseTableInfo.toList(), exception); - } - } + // Copy names under the MV lock, but resolve external metadata after releasing it: + // catalog retirement holds the catalog monitor while invalidating MV rewrite caches. + baseTableQualifiers = mtmv.getRelation().getBaseTables().stream() + .filter(BaseTableInfo::isValid) + .map(BaseTableInfo::toList) + .collect(Collectors.toList()); } finally { mtmv.readMvUnlock(); } + for (List baseTableQualifier : baseTableQualifiers) { + if (isDebugEnabled) { + LOG.debug("mtmv {} related base table include {}", new BaseTableInfo(mtmv), baseTableQualifier); + } + try { + // Collect all base tables and lock them before querying + cascadesContext.getStatementContext().getAndCacheTable(baseTableQualifier, + TableFrom.MTMV, Optional.empty()); + } catch (AnalysisException exception) { + LOG.warn("mtmv related base table get err, related table is {}", baseTableQualifier, exception); + } + } } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 5ad8af21e38261..f6e396b746da61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -49,6 +49,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.DebugPointUtil; @@ -356,6 +357,8 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_DROP_DB: { DropDbInfo dropDbInfo = (DropDbInfo) journal.getData(); env.replayDropDb(dropDbInfo); + env.onDatabaseChanged(Strings.isNullOrEmpty(dropDbInfo.getCtlName()) + ? InternalCatalog.INTERNAL_CATALOG_NAME : dropDbInfo.getCtlName(), dropDbInfo.getDbName()); break; } case OperationType.OP_ALTER_DB: { @@ -373,6 +376,8 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_RECOVER_DB: { RecoverInfo info = (RecoverInfo) journal.getData(); env.replayRecoverDatabase(info); + env.onDatabaseChanged(InternalCatalog.INTERNAL_CATALOG_NAME, + env.getInternalCatalog().getDbOrMetaException(info.getDbId()).getFullName()); break; } case OperationType.OP_RENAME_DB: { @@ -380,6 +385,8 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { String dbName = dbInfo.getDbName(); LOG.info("Begin to unprotect rename db {}", dbName); env.replayRenameDatabase(dbName, dbInfo.getNewDbName()); + env.onDatabaseChanged(InternalCatalog.INTERNAL_CATALOG_NAME, dbName); + env.onDatabaseChanged(InternalCatalog.INTERNAL_CATALOG_NAME, dbInfo.getNewDbName()); break; } case OperationType.OP_CREATE_TABLE: { @@ -388,8 +395,12 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { env.replayCreateTable(info); if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName().equals( InternalCatalog.INTERNAL_CATALOG_NAME)) { + // Legacy internal journals can omit the top-level table name. + env.onTableChanged(info.getTable()); CreateTableRecord record = new CreateTableRecord(logId, info); env.getBinlogManager().addCreateTableRecord(record); + } else { + env.onTableChanged(info.getCtlName(), info.getDbName(), info.getTblName()); } break; } @@ -398,6 +409,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { LOG.info("Begin to unprotect alter external table schema. db = {} table = {}", info.getDbName(), info.getTableName()); env.replayAlterExternalTableSchema(info.getDbName(), info.getTableName(), info.getNewSchema()); + env.onTableChanged(InternalCatalog.INTERNAL_CATALOG_NAME, info.getDbName(), info.getTableName()); break; } case OperationType.OP_DROP_TABLE: { @@ -414,6 +426,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { .getCatalog(info.getCtl()); if (ctl != null) { ctl.replayDropTable(info.getDb(), info.getTableName(), info.getResolvedDb()); + env.onTableChanged(info.getCtl(), info.getDb(), info.getTableName()); } } break; @@ -425,6 +438,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { + " partitionName = " + info.getPartition().getName()); AddPartitionRecord addPartitionRecord = new AddPartitionRecord(logId, info); env.replayAddPartition(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addAddPartitionRecord(addPartitionRecord); break; } @@ -442,6 +456,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { + info.getTableId() + " partitionId = " + info.getPartitionId()); BatchModifyPartitionsInfo infos = new BatchModifyPartitionsInfo(info); env.getAlterInstance().replayModifyPartition(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addModifyPartitions(infos, logId); break; } @@ -450,6 +465,9 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { for (ModifyPartitionInfo modifyPartitionInfo : info.getModifyPartitionInfos()) { env.getAlterInstance().replayModifyPartition(modifyPartitionInfo); } + info.getModifyPartitionInfos().stream() + .map(partition -> Pair.of(partition.getDbId(), partition.getTableId())) + .distinct().forEach(table -> env.onTableChanged(table.first, table.second)); env.getBinlogManager().addModifyPartitions(info, logId); break; } @@ -466,12 +484,14 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_RECOVER_TABLE: { RecoverInfo info = (RecoverInfo) journal.getData(); env.replayRecoverTable(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addRecoverTableRecord(info, logId); break; } case OperationType.OP_RECOVER_PARTITION: { RecoverInfo info = (RecoverInfo) journal.getData(); env.replayRecoverPartition(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addRecoverTableRecord(info, logId); break; } @@ -484,18 +504,21 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_VIEW_DEF: { AlterViewInfo info = (AlterViewInfo) journal.getData(); env.getAlterInstance().replayModifyViewDef(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addModifyViewDef(info, logId); break; } case OperationType.OP_RENAME_PARTITION: { TableInfo info = (TableInfo) journal.getData(); env.replayRenamePartition(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addPartitionRename(info, logId); break; } case OperationType.OP_RENAME_COLUMN: { TableRenameColumnInfo info = (TableRenameColumnInfo) journal.getData(); env.replayRenameColumn(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addColumnRename(info, logId); break; } @@ -513,6 +536,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_DROP_ROLLUP: { DropInfo info = (DropInfo) journal.getData(); env.getMaterializedViewHandler().replayDropRollup(info, env); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addDropRollup(info, logId); break; } @@ -534,6 +558,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { env.getMaterializedViewHandler().replayDropRollup(info, env); } } + env.onTableChanged(batchDropInfo.getDbId(), batchDropInfo.getTableId()); break; } case OperationType.OP_FINISH_CONSISTENCY_CHECK: { @@ -544,6 +569,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_RENAME_ROLLUP: { TableInfo info = (TableInfo) journal.getData(); env.replayRenameRollup(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addRollupRename(info, logId); break; } @@ -819,6 +845,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_TABLE_COLOCATE: { final TablePropertyInfo info = (TablePropertyInfo) journal.getData(); env.replayModifyTableColocate(info); + env.onTableChanged(info.getDbId(), info.getTableId()); break; } case OperationType.OP_HEARTBEAT: { @@ -994,6 +1021,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_DISTRIBUTION_TYPE: { TableInfo tableInfo = (TableInfo) journal.getData(); env.replayConvertDistributionType(tableInfo); + env.onTableChanged(tableInfo.getDbId(), tableInfo.getTableId()); env.getBinlogManager().addModifyDistributionType(tableInfo, logId); break; } @@ -1010,6 +1038,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { ModifyTableDefaultDistributionBucketNumOperationLog log = (ModifyTableDefaultDistributionBucketNumOperationLog) journal.getData(); env.replayModifyTableDefaultDistributionBucketNum(log); + env.onTableChanged(log.getDbId(), log.getTableId()); env.getBinlogManager().addModifyDistributionNum(log, logId); break; } @@ -1017,6 +1046,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { ReplacePartitionOperationLog replaceTempPartitionLog = (ReplacePartitionOperationLog) journal.getData(); env.replayReplaceTempPartition(replaceTempPartitionLog); + env.onTableChanged(replaceTempPartitionLog.getDbId(), replaceTempPartitionLog.getTblId()); env.getBinlogManager().addReplacePartitions(replaceTempPartitionLog, logId); break; } @@ -1057,6 +1087,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_COMMENT: { ModifyCommentOperationLog operation = (ModifyCommentOperationLog) journal.getData(); env.getAlterInstance().replayModifyComment(operation); + env.onTableChanged(operation.getDbId(), operation.getTblId()); env.getBinlogManager().addModifyComment(operation, logId); break; } @@ -1125,6 +1156,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_TABLE_ENGINE: { ModifyTableEngineOperationLog log = (ModifyTableEngineOperationLog) journal.getData(); env.getAlterInstance().replayProcessModifyEngine(log); + env.onTableChanged(log.getDbId(), log.getTableId()); break; } case OperationType.OP_CREATE_POLICY: { @@ -1175,12 +1207,14 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_TABLE_LIGHT_SCHEMA_CHANGE: { final TableAddOrDropColumnsInfo info = (TableAddOrDropColumnsInfo) journal.getData(); env.getSchemaChangeHandler().replayModifyTableLightSchemaChange(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addModifyTableAddOrDropColumns(info, logId); break; } case OperationType.OP_ALTER_LIGHT_SCHEMA_CHANGE: { final AlterLightSchemaChangeInfo info = (AlterLightSchemaChangeInfo) journal.getData(); env.getSchemaChangeHandler().replayAlterLightSchChange(info); + env.onTableChanged(info.getDbId(), info.getTableId()); break; } case OperationType.OP_CLEAN_QUERY_STATS: { @@ -1192,12 +1226,16 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { final TableAddOrDropInvertedIndicesInfo info = (TableAddOrDropInvertedIndicesInfo) journal.getData(); env.getSchemaChangeHandler().replayModifyTableAddOrDropInvertedIndices(info); + env.onTableChanged(info.getDbId(), info.getTableId()); env.getBinlogManager().addModifyTableAddOrDropInvertedIndices(info, logId); break; } case OperationType.OP_INVERTED_INDEX_JOB: { IndexChangeJob indexChangeJob = (IndexChangeJob) journal.getData(); env.getSchemaChangeHandler().replayIndexChangeJob(indexChangeJob); + if (indexChangeJob.getJobState() == IndexChangeJob.JobState.FINISHED) { + env.onTableChanged(indexChangeJob.getDbId(), indexChangeJob.getTableId()); + } env.getBinlogManager().addIndexChangeJob(indexChangeJob, logId); break; } @@ -1387,6 +1425,8 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_ALTER_MTMV: { final AlterMTMV alterMtmv = (AlterMTMV) journal.getData(); env.getAlterInstance().processAlterMTMV(alterMtmv, true); + env.onTableChanged(alterMtmv.getMvName().getCtl(), + alterMtmv.getMvName().getDb(), alterMtmv.getMvName().getTbl()); break; } case OperationType.OP_INSERT_OVERWRITE: { @@ -1455,6 +1495,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { CatalogIf ctl = Env.getCurrentEnv().getCatalogMgr().getCatalog(info.getCtlName()); if (ctl != null) { ctl.replayOperateOnBranchOrTag(info.getDbName(), info.getTblName()); + env.onTableChanged(info.getCtlName(), info.getDbName(), info.getTblName()); } break; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java index ff2772e948200b..8cf0ed41be6860 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java @@ -20,6 +20,7 @@ import org.apache.doris.alter.AlterJobV2.JobState; import org.apache.doris.analysis.AddRollupClause; import org.apache.doris.analysis.AlterClause; +import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.CatalogTestUtil; import org.apache.doris.catalog.Column; @@ -44,6 +45,7 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.info.TableNameInfo; import org.apache.doris.meta.MetaContext; +import org.apache.doris.nereids.SqlCacheContext; import org.apache.doris.nereids.trees.plans.commands.CancelAlterTableCommand; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.OriginStatement; @@ -278,8 +280,14 @@ public void testSchemaChange1() throws Exception { } } + // A query can populate a cache while the shadow index is still invisible. + SqlCacheContext cached = new SqlCacheContext(UserIdentity.ROOT); + cached.addUsedTable(olapTable); + Assert.assertFalse(cached.getUsedTables().isEmpty()); + masterEnv.getSqlCacheManager().getSqlCaches().put("during_rollup", cached); materializedViewHandler.runAfterCatalogReady(); Assert.assertEquals(JobState.FINISHED, rollupJob.getJobState()); + Assert.assertNull(masterEnv.getSqlCacheManager().getSqlCaches().getIfPresent("during_rollup")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/MTMVRewriteCacheInvalidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/MTMVRewriteCacheInvalidationTest.java new file mode 100644 index 00000000000000..c1b1bcc824a9ba --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/MTMVRewriteCacheInvalidationTest.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.mtmv.MTMVCache; +import org.apache.doris.mtmv.MTMVRefreshSnapshot; +import org.apache.doris.mtmv.MTMVStatus; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mockito; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +class MTMVRewriteCacheInvalidationTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void invalidationRejectsAnOlderBuildForBothCacheVariants(boolean sameSessionVariables) throws Exception { + CountDownLatch firstBuild = new CountDownLatch(1); + CountDownLatch finishBuild = new CountDownLatch(1); + AtomicInteger builds = new AtomicInteger(); + MTMVCache old = Mockito.mock(MTMVCache.class); + MTMVCache fresh = Mockito.mock(MTMVCache.class); + MTMV mv = new MTMV() { + @Override + protected MTMVCache createRewriteCache(ConnectContext context, boolean needLock, boolean addGuard) { + Assertions.assertEquals(!sameSessionVariables, addGuard); + if (builds.incrementAndGet() == 1) { + firstBuild.countDown(); + try { + Assertions.assertTrue(finishBuild.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + return old; + } + return fresh; + } + }; + Deencapsulation.setField(mv, "status", new MTMVStatus()); + Deencapsulation.setField(mv, "refreshSnapshot", new MTMVRefreshSnapshot()); + if (!sameSessionVariables) { + Deencapsulation.setField(mv, "sessionVariables", ImmutableMap.of("test", "different")); + } + ConnectContext connection = Mockito.mock(ConnectContext.class); + Mockito.when(connection.getSessionVariable()).thenReturn(new SessionVariable()); + ExecutorService worker = Executors.newSingleThreadExecutor(); + try { + Future result = worker.submit(() -> mv.getOrGenerateCache(connection)); + Assertions.assertTrue(firstBuild.await(5, TimeUnit.SECONDS)); + mv.invalidateRewriteCache(); + finishBuild.countDown(); + Assertions.assertSame(fresh, result.get(5, TimeUnit.SECONDS)); + Assertions.assertSame(fresh, mv.getOrGenerateCache(connection)); + Assertions.assertEquals(2, builds.get()); + } finally { + finishBuild.countDown(); + worker.shutdownNow(); + Assertions.assertTrue(worker.awaitTermination(5, TimeUnit.SECONDS)); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/cache/SqlCacheMetadataVersionsTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/cache/SqlCacheMetadataVersionsTest.java new file mode 100644 index 00000000000000..2003c44558d64d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/common/cache/SqlCacheMetadataVersionsTest.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.cache; + +import org.apache.doris.nereids.SqlCacheContext.FullTableName; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class SqlCacheMetadataVersionsTest { + private final SqlCacheMetadataVersions versions = new SqlCacheMetadataVersions(); + private final FullTableName table = new FullTableName("catalog", "db", "table"); + + private SqlCacheMetadataVersions.Snapshot snapshot(long id, FullTableName name) { + SqlCacheMetadataVersions.Snapshot snapshot = versions.snapshot(); + snapshot.addTable(id, name); + return snapshot; + } + + @ParameterizedTest + @CsvSource({"id", "name", "database", "catalog", "all"}) + void affectedDependenciesChange(String scope) { + SqlCacheMetadataVersions.Snapshot affected = snapshot(1L, table); + SqlCacheMetadataVersions.Snapshot other = snapshot(2L, new FullTableName("other", "db", "table")); + switch (scope) { + case "id": + versions.invalidateTable(1L, null); + break; + case "name": + versions.invalidateTable(-1L, table); + break; + case "database": + versions.invalidateNamespace("catalog", "db"); + break; + case "catalog": + versions.invalidateNamespace("catalog", null); + break; + case "all": + versions.invalidateAll(); + break; + default: + throw new AssertionError(scope); + } + Assertions.assertFalse(affected.isCurrent()); + Assertions.assertEquals(!scope.equals("all"), other.isCurrent()); + Assertions.assertTrue(snapshot(1L, table).isCurrent()); + } + + @Test + void tableChangePreservesOtherTablesInTheSameNamespace() { + SqlCacheMetadataVersions.Snapshot target = snapshot(1L, table); + SqlCacheMetadataVersions.Snapshot other = snapshot(2L, new FullTableName("catalog", "db", "other")); + versions.invalidateTable(1L, table); + Assertions.assertFalse(target.isCurrent()); + Assertions.assertTrue(other.isCurrent()); + } + + @Test + void dependencyRecordedAfterRefreshCannotAdoptANewRevision() { + SqlCacheMetadataVersions.Snapshot oldQuery = versions.snapshot(); + versions.invalidateNamespace("catalog", "db"); + oldQuery.addTable(1L, table); + Assertions.assertFalse(oldQuery.isCurrent()); + Assertions.assertTrue(snapshot(1L, table).isCurrent()); + } + + @Test + void recordingTheSameTableAgainCannotReviveAnOldDependency() { + SqlCacheMetadataVersions.Snapshot oldQuery = snapshot(1L, table); + versions.invalidateTable(1L, table); + oldQuery.addTable(1L, table); + Assertions.assertFalse(oldQuery.isCurrent()); + } + + @Test + void globalInvalidationIncludesQueriesWithoutTables() { + SqlCacheMetadataVersions.Snapshot constant = versions.snapshot(); + versions.invalidateAll(); + Assertions.assertFalse(constant.isCurrent()); + Assertions.assertTrue(versions.snapshot().isCurrent()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index 598edb8995f1bc..208e52b66b4ab9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -606,8 +606,10 @@ public void invalidateTable(long id, String dbName, String tableName) { cacheMgr.replaceRowCountCacheForTest(rowCountCache); HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("hms"); ExternalDatabase db = Mockito.mock(ExternalDatabase.class); Mockito.when(db.getId()).thenReturn(dbId); + Mockito.when(db.getFullName()).thenReturn("db"); ExternalTable table = Mockito.mock(ExternalTable.class); Mockito.when(table.getCatalog()).thenReturn(catalog); Mockito.when(table.getDb()).thenReturn(db); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalQueryCacheInvalidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalQueryCacheInvalidationTest.java new file mode 100644 index 00000000000000..baa60d976c104f --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalQueryCacheInvalidationTest.java @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; +import org.apache.doris.common.cache.NereidsSqlCacheManager; +import org.apache.doris.common.cache.SqlCacheMetadataVersions; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.metacache.ExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheRegistry; +import org.apache.doris.datasource.metacache.ExternalMetaCacheRouteResolver; +import org.apache.doris.mtmv.MTMVRelationManager; +import org.apache.doris.mtmv.MTMVService; +import org.apache.doris.nereids.SqlCacheContext.FullTableName; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.concurrent.atomic.AtomicBoolean; + +class ExternalQueryCacheInvalidationTest { + @ParameterizedTest + @CsvSource({"catalog,false", "catalog,true", "remove,false", "remove,true", + "context,false", "context,true", "database,false", "database,true", + "table,false", "table,true", "typed,false", "typed,true", + "engine,false", "engine,true", "partitions,false", "partitions,true"}) + void metadataInvalidationFencesQueryDependenciesAndKeepsTheFinalRowCountFence( + String operation, boolean failure) { + Env env = Mockito.mock(Env.class, Mockito.CALLS_REAL_METHODS); + CatalogMgr catalogs = Mockito.mock(CatalogMgr.class); + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getName()).thenReturn("lake"); + Mockito.when(catalog.getId()).thenReturn(1L); + Mockito.doReturn(catalog).when(catalogs).getCatalog(1L); + Mockito.doReturn(catalogs).when(env).getCatalogMgr(); + NereidsSqlCacheManager sql = new NereidsSqlCacheManager(); + Mockito.doReturn(sql).when(env).getSqlCacheManager(); + Mockito.doReturn(Mockito.mock(NereidsSortedPartitionsCacheManager.class)) + .when(env).getSortedPartitionsCacheManager(); + MTMVRelationManager relations = Mockito.mock(MTMVRelationManager.class); + MTMVService service = Mockito.mock(MTMVService.class); + Mockito.when(service.getRelationManager()).thenReturn(relations); + Mockito.doReturn(service).when(env).getMtmvService(); + SqlCacheMetadataVersions.Snapshot affected = sql.newMetadataSnapshot(); + affected.addTable(3L, new FullTableName("lake", "db", "t")); + SqlCacheMetadataVersions.Snapshot unrelated = sql.newMetadataSnapshot(); + unrelated.addTable(4L, new FullTableName("other", "db", "t")); + ExternalDatabase database = Mockito.mock(ExternalDatabase.class); + Mockito.doReturn(catalog).when(database).getCatalog(); + Mockito.when(database.getId()).thenReturn(2L); + Mockito.when(database.getFullName()).thenReturn("db"); + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.doReturn(catalog).when(table).getCatalog(); + Mockito.doReturn(database).when(table).getDb(); + Mockito.when(table.getId()).thenReturn(3L); + Mockito.when(table.getName()).thenReturn("t"); + + try (MockedStatic current = Mockito.mockStatic(Env.class)) { + current.when(Env::getCurrentEnv).thenReturn(env); + ExternalMetaCacheMgr caches = new ExternalMetaCacheMgr(true); + AtomicBoolean closingFence = new AtomicBoolean(); + ExternalRowCountCache rows = Mockito.mock(ExternalRowCountCache.class, invocation -> { + if (!affected.isCurrent()) { + closingFence.set(true); + } + return Mockito.RETURNS_DEFAULTS.answer(invocation); + }); + Deencapsulation.setField(caches, "rowCountCache", rows); + ExternalMetaCache engine = Mockito.mock(ExternalMetaCache.class, invocation -> { + String method = invocation.getMethod().getName(); + if (method.equals("engine")) { + return "test"; + } + if (method.equals("isCatalogInitialized")) { + return true; + } + if (method.startsWith("invalidate")) { + Assertions.assertTrue(affected.isCurrent(), "engine invalidation precedes derived invalidation"); + if (failure) { + throw new IllegalStateException("engine failure"); + } + } + return Mockito.RETURNS_DEFAULTS.answer(invocation); + }); + ExternalMetaCacheRegistry registry = Mockito.mock(ExternalMetaCacheRegistry.class); + Mockito.when(registry.resolve("test")).thenReturn(engine); + Deencapsulation.setField(caches, "cacheRegistry", registry); + ExternalMetaCacheRouteResolver routes = Mockito.mock(ExternalMetaCacheRouteResolver.class); + Mockito.when(routes.resolveCatalogCaches(1L, catalog)).thenReturn(ImmutableList.of(engine)); + Deencapsulation.setField(caches, "routeResolver", routes); + Runnable invalidate = () -> { + switch (operation) { + case "catalog": + caches.invalidateCatalog(1L); + break; + case "remove": + caches.removeCatalog(1L, "lake"); + break; + case "context": + caches.onCatalogOperationalContextChanged(1L); + break; + case "database": + caches.invalidateDb(1L, 2L, "db"); + break; + case "table": + caches.invalidateTable(1L, "db", "t"); + break; + case "typed": + caches.invalidateTableCache(table); + break; + case "engine": + caches.invalidateTableByEngine(1L, "test", "db", "t"); + break; + case "partitions": + caches.invalidatePartitions(1L, "db", "t", ImmutableList.of("p")); + break; + default: + throw new AssertionError(operation); + } + }; + if (failure) { + Assertions.assertThrows(IllegalStateException.class, invalidate::run); + } else { + invalidate.run(); + } + Assertions.assertFalse(affected.isCurrent()); + Assertions.assertTrue(unrelated.isCurrent()); + Mockito.verify(relations).invalidateRewriteCaches(Mockito.eq("lake"), Mockito.any(), Mockito.any()); + // Every scope must leave row counts fenced after the derived-cache callback completes. + Assertions.assertTrue(closingFence.get(), "row-count invalidation must follow derived invalidation"); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java index 7e549056b2ee7a..01c48e9bb94e00 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; @@ -139,6 +140,7 @@ public void testPausedTvfJobValidatesAlterOffsetAfterRestore() throws Exception ConnectContext previousContext = ConnectContext.get(); try (MockedStatic envMockedStatic = Mockito.mockStatic(Env.class)) { Env env = Mockito.mock(Env.class); + Mockito.when(env.getSqlCacheManager()).thenReturn(new NereidsSqlCacheManager()); InternalCatalog catalog = Mockito.mock(InternalCatalog.class); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); Mockito.when(catalog.getName()).thenReturn(InternalCatalog.INTERNAL_CATALOG_NAME); diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskAuditTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskAuditTest.java index e2f81e4221bb8f..7158dbe8208b7c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskAuditTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertTaskAuditTest.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.StmtType; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Env; +import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.datasource.CatalogMgr; @@ -328,6 +329,7 @@ private AuditEvent runS3Task(RuntimeException commandFailure) throws Exception { private AuditEvent runTask(RuntimeException commandFailure, String sql, SourceOffsetProvider offsetProvider, Offset offset, boolean expectAudit) throws Exception { Env env = Mockito.mock(Env.class); + Mockito.when(env.getSqlCacheManager()).thenReturn(new NereidsSqlCacheManager()); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); InternalCatalog catalog = Mockito.mock(InternalCatalog.class); WorkloadRuntimeStatusMgr statusMgr = Mockito.mock(WorkloadRuntimeStatusMgr.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationCacheInvalidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationCacheInvalidationTest.java new file mode 100644 index 00000000000000..bd287777b7edb7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationCacheInvalidationTest.java @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.info.TableNameInfo; + +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +class MTMVRelationCacheInvalidationTest { + @Test + void tableInvalidationUsesTheExistingGenerationBarrier() { + MTMVRelationManager manager = new MTMVRelationManager(); + BaseTableInfo source = name("lake", "source", "t"); + addDependency(manager, source, "dependent"); + addDependency(manager, name("lake", "source", "other"), "unrelated"); + MTMV dependent = Mockito.mock(MTMV.class); + MTMV unrelated = Mockito.mock(MTMV.class); + try (MockedStatic env = localViews(dependent, unrelated)) { + manager.invalidateRewriteCaches("lake", "source", "t"); + Mockito.verify(dependent).invalidateRewriteCache(); + Mockito.verify(unrelated, Mockito.never()).invalidateRewriteCache(); + } + } + + @Test + void namespaceInvalidationDoesNotRequireWarmBaseObjects() { + MTMVRelationManager manager = new MTMVRelationManager(); + addDependency(manager, name("lake", "first", "t"), "dependent"); + addDependency(manager, name("lake", "second", "t"), "unrelated"); + MTMV first = Mockito.mock(MTMV.class); + MTMV second = Mockito.mock(MTMV.class); + try (MockedStatic env = localViews(first, second)) { + manager.invalidateRewriteCaches("lake", "first", null); + Mockito.verify(first).invalidateRewriteCache(); + Mockito.verify(second, Mockito.never()).invalidateRewriteCache(); + manager.invalidateRewriteCaches("lake", null, null); + Mockito.verify(first, Mockito.times(2)).invalidateRewriteCache(); + Mockito.verify(second).invalidateRewriteCache(); + } + } + + @Test + void oneFailingViewDoesNotBlockTheOtherViews() { + MTMVRelationManager manager = new MTMVRelationManager(); + BaseTableInfo source = name("lake", "source", "t"); + addDependency(manager, source, "dependent"); + addDependency(manager, source, "unrelated"); + MTMV failing = Mockito.mock(MTMV.class); + MTMV healthy = Mockito.mock(MTMV.class); + Mockito.doThrow(new IllegalStateException("cache failure")).when(failing).invalidateRewriteCache(); + try (MockedStatic env = localViews(failing, healthy)) { + manager.invalidateRewriteCaches("lake", "source", "t"); + Mockito.verify(failing).invalidateRewriteCache(); + Mockito.verify(healthy).invalidateRewriteCache(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"tableMTMVs", "viewMTMVs", "tableMTMVsOneLevelAndFromView"}) + void concurrentRegistrationKeepsBothDependentViews(String indexField) throws Exception { + MTMVRelationManager manager = new MTMVRelationManager(); + FirstRegistrationMap index = new FirstRegistrationMap(); + Deencapsulation.setField(manager, indexField, index); + BaseTableInfo source = name("lake", "source", "t"); + Set tables = indexField.equals("viewMTMVs") ? ImmutableSet.of() : ImmutableSet.of(source); + Set views = indexField.equals("viewMTMVs") ? ImmutableSet.of(source) : ImmutableSet.of(); + MTMVRelation relation = new MTMVRelation(tables, tables, tables, views, views); + BaseTableInfo first = name("internal", "views", "dependent"); + BaseTableInfo second = name("internal", "views", "unrelated"); + ExecutorService workers = Executors.newFixedThreadPool(2); + try { + Future firstRegistration = workers.submit(() -> register(manager, relation, first, index)); + Future secondRegistration = workers.submit(() -> register(manager, relation, second, index)); + firstRegistration.get(10, TimeUnit.SECONDS); + secondRegistration.get(10, TimeUnit.SECONDS); + } finally { + workers.shutdownNow(); + Assertions.assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + Assertions.assertEquals(ImmutableSet.of(first, second), index.get(source), + "concurrent first registrations must not replace another MV's dependency set"); + if (!indexField.equals("tableMTMVsOneLevelAndFromView")) { + MTMV firstView = Mockito.mock(MTMV.class); + MTMV secondView = Mockito.mock(MTMV.class); + try (MockedStatic ignored = localViews(firstView, secondView)) { + manager.invalidateRewriteCaches("lake", "source", "t"); + Mockito.verify(firstView).invalidateRewriteCache(); + Mockito.verify(secondView).invalidateRewriteCache(); + } + } + } + + private void register(MTMVRelationManager manager, MTMVRelation relation, + BaseTableInfo view, FirstRegistrationMap index) { + try { + manager.refreshMTMVCache(relation, view); + } finally { + index.firstRegistered.countDown(); + } + } + + private static class FirstRegistrationMap extends ConcurrentHashMap> { + private static final long serialVersionUID = 1L; + private final AtomicInteger misses = new AtomicInteger(); + private final CountDownLatch secondMiss = new CountDownLatch(1); + private final CountDownLatch firstRegistered = new CountDownLatch(1); + + @Override + public boolean containsKey(Object key) { + boolean present = super.containsKey(key); + if (!present) { + // Both old check-then-put callers observe absence. Let one finish registering before + // the other resumes its stale miss. Atomic computeIfAbsent does not take this path. + if (misses.incrementAndGet() == 1) { + await(secondMiss); + } else { + secondMiss.countDown(); + await(firstRegistered); + } + } + return present; + } + + private static void await(CountDownLatch latch) { + try { + Assertions.assertTrue(latch.await(5, TimeUnit.SECONDS), "registration interleaving timed out"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } + } + + private BaseTableInfo name(String catalog, String db, String table) { + return new BaseTableInfo(new TableNameInfo(catalog, db, table)); + } + + private void addDependency(MTMVRelationManager manager, BaseTableInfo source, String view) { + manager.refreshMTMVCache(new MTMVRelation(ImmutableSet.of(source), ImmutableSet.of(source), + ImmutableSet.of(source), ImmutableSet.of(), ImmutableSet.of()), + name("internal", "views", view)); + } + + private MockedStatic localViews(MTMV dependent, MTMV unrelated) { + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + Mockito.when(catalog.getDbNullable("views")).thenReturn(database); + Mockito.when(database.getTableNullable("dependent")).thenReturn(dependent); + Mockito.when(database.getTableNullable("unrelated")).thenReturn(unrelated); + MockedStatic env = Mockito.mockStatic(Env.class); + env.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + return env; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java index 3ea04dac288369..b767f929b1a5c2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java @@ -21,8 +21,10 @@ import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; +import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalScanTaskCacheKey; import org.apache.doris.datasource.ExternalTable; @@ -43,6 +45,7 @@ import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; import org.mockito.InOrder; +import org.mockito.MockedStatic; import org.mockito.Mockito; import java.io.Closeable; @@ -63,6 +66,48 @@ public class StatementContextTest { + @Test + public void testIdGeneratorContextDoesNotCaptureCurrentConnection() { + ConnectContext previousContext = ConnectContext.get(); + ConnectContext connection = new ConnectContext(); + connection.setThreadLocalInfo(); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + StatementContext statement = new StatementContext(10000); + org.junit.jupiter.api.Assertions.assertNull(statement.getConnectContext()); + org.junit.jupiter.api.Assertions.assertFalse(statement.getSqlCacheContext().isPresent()); + org.junit.jupiter.api.Assertions.assertEquals(10000, statement.getNextExprId().asInt()); + org.junit.jupiter.api.Assertions.assertEquals(10001, statement.getNextExprId().asInt()); + mockedEnv.verifyNoInteractions(); + } finally { + ConnectContext.remove(); + if (previousContext != null) { + previousContext.setThreadLocalInfo(); + } + } + } + + @Test + public void testSqlCacheSnapshotIsFreshForEachPreparedExecution() { + Env env = Mockito.mock(Env.class); + NereidsSqlCacheManager manager = new NereidsSqlCacheManager(); + Mockito.when(env.getSqlCacheManager()).thenReturn(manager); + ConnectContext connection = new ConnectContext(); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + StatementContext statement = new StatementContext(connection, new OriginStatement("select 1", 0)); + SqlCacheContext firstExecution = statement.getSqlCacheContext().orElseThrow(AssertionError::new); + org.junit.jupiter.api.Assertions.assertTrue(firstExecution.metadataVersionsMatch()); + + manager.invalidateAll(); + org.junit.jupiter.api.Assertions.assertFalse(firstExecution.metadataVersionsMatch()); + SqlCacheContext nextExecution = statement.createNextExecuteContext().getSqlCacheContext() + .orElseThrow(AssertionError::new); + org.junit.jupiter.api.Assertions.assertTrue(nextExecution.metadataVersionsMatch()); + manager.invalidateAll(); + org.junit.jupiter.api.Assertions.assertFalse(nextExecution.metadataVersionsMatch()); + } + } + @Test public void testPreloadedPartitionViewSurvivesPlainHiveBindingAndResets() { StatementContext statement = new StatementContext(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CollectRelationMTMVLockTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CollectRelationMTMVLockTest.java new file mode 100644 index 00000000000000..32c86b1c5ecfe5 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CollectRelationMTMVLockTest.java @@ -0,0 +1,313 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.analysis; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; +import org.apache.doris.common.cache.NereidsSqlCacheManager; +import org.apache.doris.common.cache.SqlCacheMetadataVersions; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalMetaCacheMgr; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.InitCatalogLog; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.SessionContext; +import org.apache.doris.info.TableNameInfo; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVCache; +import org.apache.doris.mtmv.MTMVRelation; +import org.apache.doris.mtmv.MTMVRelationManager; +import org.apache.doris.mtmv.MTMVService; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.SqlCacheContext.FullTableName; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.StatementContext.TableFrom; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +class CollectRelationMTMVLockTest { + @Test + void catalogRetirementCompletesDuringCandidateBaseLookup() throws Exception { + Env env = Mockito.mock(Env.class, Mockito.CALLS_REAL_METHODS); + CatalogMgr catalogs = Mockito.mock(CatalogMgr.class); + LookupCatalog catalog = new LookupCatalog(); + ExternalDatabase database = Mockito.mock(ExternalDatabase.class); + ExternalTable base = Mockito.mock(ExternalTable.class); + Mockito.doReturn(database).when(base).getDatabase(); + Mockito.when(base.getName()).thenReturn("base"); + Mockito.when(base.getId()).thenReturn(3L); + Mockito.doReturn(catalog).when(database).getCatalog(); + Mockito.when(database.getFullName()).thenReturn("source"); + Mockito.doReturn(base).when(database).getTableNullable("base"); + catalog.database = database; + Mockito.doReturn(catalog).when(catalogs).getCatalog("lake"); + Mockito.doReturn(catalog).when(catalogs).getCatalog(1L); + Mockito.doReturn(catalogs).when(env).getCatalogMgr(); + AccessControllerManager access = Mockito.mock(AccessControllerManager.class); + Mockito.doReturn(access).when(env).getAccessManager(); + Mockito.doReturn(new ExternalMetaCacheMgr(true)).when(env).getExtMetaCacheMgr(); + NereidsSqlCacheManager sqlCaches = new NereidsSqlCacheManager(); + Mockito.doReturn(sqlCaches).when(env).getSqlCacheManager(); + Mockito.doReturn(Mockito.mock(NereidsSortedPartitionsCacheManager.class)) + .when(env).getSortedPartitionsCacheManager(); + MTMV mv = view(ImmutableSet.of(name("base"))); + MTMVRelationManager relations = Mockito.spy(new MTMVRelationManager()); + relations.refreshMTMVCache(mv.getRelation(), + new BaseTableInfo(new TableNameInfo("internal", "views", "mv"))); + Mockito.doReturn(ImmutableSet.of(mv)).when(relations).getCandidateMTMVs(Mockito.anyList()); + Mockito.doAnswer(invocation -> { + Assertions.assertFalse(Thread.holdsLock(catalog), "derived invalidation must run outside the catalog monitor"); + return invocation.callRealMethod(); + }).when(relations).invalidateRewriteCaches("lake", null, null); + MTMVService service = Mockito.mock(MTMVService.class); + Mockito.when(service.getRelationManager()).thenReturn(relations); + Mockito.doReturn(service).when(env).getMtmvService(); + InternalCatalog internal = Mockito.mock(InternalCatalog.class); + Database views = Mockito.mock(Database.class); + Mockito.when(internal.getDbNullable("views")).thenReturn(views); + Mockito.when(views.getTableNullable("mv")).thenReturn(mv); + ConnectContext connection = Mockito.mock(ConnectContext.class); + Mockito.when(connection.getEnv()).thenReturn(env); + Mockito.when(connection.getSessionVariable()).thenReturn(new SessionVariable()); + Mockito.when(connection.getCurrentUserIdentity()).thenReturn(UserIdentity.ROOT); + AtomicReference resetFailure = new AtomicReference<>(); + Thread reset = new Thread(() -> { + try (MockedStatic ignored = localEnv(env, internal)) { + Assertions.assertTrue(catalog.lookupStarted.await(5, TimeUnit.SECONDS)); + catalog.resetToUninitialized(false); + } catch (Throwable failure) { + resetFailure.set(failure); + } finally { + catalog.retirementFinished.countDown(); + } + }, "catalog-retirement-during-mv-collection"); + reset.setDaemon(true); + try (MockedStatic ignored = localEnv(env, internal)) { + StatementContext statement = new StatementContext(connection, null); + List qualifier = name("base").toList(); + // QUERY and MTMV have separate lookup maps, even for the same single base table. + Assertions.assertSame(base, statement.getAndCacheTable(qualifier, TableFrom.QUERY, Optional.empty())); + Assertions.assertTrue(statement.getMtmvRelatedTables().isEmpty()); + SqlCacheMetadataVersions.Snapshot snapshot = sqlCaches.newMetadataSnapshot(); + snapshot.addTable(3L, new FullTableName("lake", "source", "base")); + MTMVCache cached = Mockito.mock(MTMVCache.class); + Deencapsulation.setField(mv, "cacheWithGuard", cached); + Deencapsulation.setField(mv, "cacheWithoutGuard", cached); + Assertions.assertSame(cached, Deencapsulation.getField(mv, "cacheWithGuard")); + Assertions.assertSame(cached, Deencapsulation.getField(mv, "cacheWithoutGuard")); + Assertions.assertTrue(snapshot.isCurrent()); + long generation = Deencapsulation.getField(mv, "rewriteCacheGeneration"); + catalog.pauseLookup = true; + reset.start(); + try { + collect(base, statement); + } finally { + reset.join(TimeUnit.SECONDS.toMillis(10)); + } + Assertions.assertFalse(reset.isAlive(), "catalog retirement did not finish"); + Assertions.assertNull(resetFailure.get()); + Assertions.assertFalse(catalog.retirementBlocked.get(), "MV read lock blocked catalog retirement"); + Assertions.assertSame(base, statement.getMtmvRelatedTables().get(qualifier)); + Assertions.assertNull(Deencapsulation.getField(mv, "cacheWithGuard")); + Assertions.assertNull(Deencapsulation.getField(mv, "cacheWithoutGuard")); + Assertions.assertEquals(generation + 1, (long) Deencapsulation.getField(mv, "rewriteCacheGeneration")); + Assertions.assertFalse(snapshot.isCurrent(), "retirement must still fence old SQL dependencies"); + } + } + + @Test + void dependencyNamesAreCopiedBeforeResolvingTables() throws Exception { + BaseTableInfo first = name("first"); + BaseTableInfo second = name("second"); + Set dependencies = new LinkedHashSet<>(Arrays.asList(first, second)); + MTMV mv = view(dependencies); + StatementContext statement = Mockito.spy(new StatementContext(0)); + AtomicReference mutationFailure = new AtomicReference<>(); + Thread mutate = new Thread(() -> { + mv.writeMvLock(); + try { + second.setTableName("changed"); + dependencies.clear(); + } catch (Throwable failure) { + mutationFailure.set(failure); + } finally { + mv.writeMvUnlock(); + } + }, "mv-dependency-update-during-collection"); + mutate.setDaemon(true); + TableIf collected = Mockito.mock(TableIf.class); + Mockito.doAnswer(invocation -> { + mutate.start(); + mutate.join(TimeUnit.SECONDS.toMillis(5)); + Assertions.assertFalse(mutate.isAlive(), "table resolution must not hold the MV lock"); + return collected; + }).when(statement).getAndCacheTable(first.toList(), TableFrom.MTMV, Optional.empty()); + Mockito.doReturn(collected).when(statement) + .getAndCacheTable(second.toList(), TableFrom.MTMV, Optional.empty()); + try { + collectWithCandidate(mv, statement); + } finally { + mutate.join(TimeUnit.SECONDS.toMillis(10)); + } + Assertions.assertNull(mutationFailure.get()); + Mockito.verify(statement).getAndCacheTable(ImmutableList.of("lake", "source", "second"), + TableFrom.MTMV, Optional.empty()); + } + + @Test + void invalidAndMissingDependenciesDoNotPreventCollectingOtherTables() { + BaseTableInfo invalid = name("invalid"); + invalid.setTableName(null); + MTMV mv = view(new LinkedHashSet<>(Arrays.asList(invalid, name("missing"), name("valid")))); + StatementContext statement = Mockito.spy(new StatementContext(0)); + Mockito.doThrow(new AnalysisException("missing table")).when(statement) + .getAndCacheTable(name("missing").toList(), TableFrom.MTMV, Optional.empty()); + Mockito.doReturn(Mockito.mock(TableIf.class)).when(statement) + .getAndCacheTable(name("valid").toList(), TableFrom.MTMV, Optional.empty()); + collectWithCandidate(mv, statement); + Mockito.verify(statement).getAndCacheTable(name("valid").toList(), TableFrom.MTMV, Optional.empty()); + Mockito.verify(statement, Mockito.never()) + .getAndCacheTable(invalid.toList(), TableFrom.MTMV, Optional.empty()); + } + + private void collectWithCandidate(MTMV mv, StatementContext statement) { + Env env = Mockito.mock(Env.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(env.getMtmvService().getRelationManager().getCandidateMTMVs(Mockito.anyList())) + .thenReturn(ImmutableSet.of(mv)); + ExternalTable base = Mockito.mock(ExternalTable.class); + ExternalDatabase database = Mockito.mock(ExternalDatabase.class); + Mockito.doReturn(database).when(base).getDatabase(); + Mockito.doReturn(Mockito.mock(ExternalCatalog.class)).when(database).getCatalog(); + try (MockedStatic ignored = localEnv(env, Mockito.mock(InternalCatalog.class))) { + collect(base, statement); + } + } + + private void collect(TableIf base, StatementContext statement) { + statement.addPlannerHook(InitMaterializationContextHook.INSTANCE); + CascadesContext cascades = Mockito.mock(CascadesContext.class); + Mockito.when(cascades.getStatementContext()).thenReturn(statement); + Deencapsulation.invoke(new CollectRelation(true), "collectMTMVCandidates", base, cascades); + } + + private static MockedStatic localEnv(Env env, InternalCatalog internal) { + MockedStatic scope = Mockito.mockStatic(Env.class); + scope.when(Env::getCurrentEnv).thenReturn(env); + scope.when(Env::getCurrentInternalCatalog).thenReturn(internal); + return scope; + } + + private static BaseTableInfo name(String table) { + return new BaseTableInfo(new TableNameInfo("lake", "source", table)); + } + + private static MTMV view(Set tables) { + MTMV mv = Mockito.spy(new MTMV()); + Deencapsulation.setField(mv, "name", "mv"); + mv.setRelation(relation(tables)); + Database database = Mockito.mock(Database.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Mockito.when(catalog.getName()).thenReturn("internal"); + Mockito.when(database.getFullName()).thenReturn("views"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.doReturn(database).when(mv).getDatabase(); + return mv; + } + + private static MTMVRelation relation(Set tables) { + return new MTMVRelation(tables, tables, tables, ImmutableSet.of(), ImmutableSet.of()); + } + + private static class LookupCatalog extends ExternalCatalog { + private ExternalDatabase database; + private volatile boolean pauseLookup; + private final CountDownLatch lookupStarted = new CountDownLatch(1); + private final CountDownLatch retirementFinished = new CountDownLatch(1); + private final AtomicBoolean retirementBlocked = new AtomicBoolean(); + + LookupCatalog() { + super(1L, "lake", InitCatalogLog.Type.TEST, ""); + initialized = true; + } + + @Override + public ExternalDatabase getDbNullable(String name) { + if (pauseLookup) { + lookupStarted.countDown(); + try { + // Bound the old lock cycle so a failing test releases its MV read lock. + if (!retirementFinished.await(5, TimeUnit.SECONDS)) { + retirementBlocked.set(true); + throw new IllegalStateException("MV collection blocked catalog retirement"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + makeSureInitialized(); + return database; + } + + @Override + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { + return ImmutableList.of(); + } + + @Override + public boolean tableExist(SessionContext ctx, String dbName, String tableName) { + return false; + } + + @Override + protected void initLocalObjectsImpl() { + } + + @Override + public void onRefreshCache(boolean invalidCache) { + initialized = true; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkDisabledTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkDisabledTest.java index 142f35b4da2f83..f9799a21070fca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkDisabledTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkDisabledTest.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.catalog.Env; +import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -59,6 +60,7 @@ void disabledLocalKeepsRemoteSinkValidation() { private Plan analyzeInsert(String sql, boolean authorized) { ConnectContext context = MemoTestUtils.createConnectContext(); Env env = Mockito.mock(Env.class); + Mockito.when(env.getSqlCacheManager()).thenReturn(new NereidsSqlCacheManager()); AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); Mockito.when(env.getAccessManager()).thenReturn(accessManager); Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(authorized); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/AuditLogWorkloadGroupTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/AuditLogWorkloadGroupTest.java index 4cd7dd5dff85b6..accec01867d720 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/AuditLogWorkloadGroupTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/AuditLogWorkloadGroupTest.java @@ -23,6 +23,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ConnectionException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.proto.Data; import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr; @@ -283,6 +284,7 @@ private void runResolveUnderMockEnv(ConnectContext ctx, String userGroup) { private void mockEnvAuth(MockedStatic mockedEnv, String userGroup) { Env env = Mockito.mock(Env.class); + Mockito.when(env.getSqlCacheManager()).thenReturn(new NereidsSqlCacheManager()); Auth auth = Mockito.mock(Auth.class); Mockito.when(env.getAuth()).thenReturn(auth); Mockito.when(auth.getWorkloadGroup(Mockito.anyString())).thenReturn(userGroup); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/MysqlPreparedStatementForwardingTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/MysqlPreparedStatementForwardingTest.java index 0ab8e1eee6f850..9e1cc495e257e7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/MysqlPreparedStatementForwardingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/MysqlPreparedStatementForwardingTest.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MysqlColType; +import org.apache.doris.common.cache.NereidsSqlCacheManager; import org.apache.doris.mysql.MysqlCapability; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.nereids.StatementContext; @@ -55,6 +56,7 @@ public void testForwardedExecutionsRetainTypesAndNullBitmap() throws Exception { follower.setCurrentUserIdentity(UserIdentity.ROOT); follower.setCapability(MysqlCapability.DEFAULT_CAPABILITY); Env env = Mockito.mock(Env.class); + Mockito.when(env.getSqlCacheManager()).thenReturn(new NereidsSqlCacheManager()); Mockito.when(env.getSelfNode()).thenReturn(new SystemInfoService.HostInfo("127.0.0.1", 9010)); PreparedStatementContext prepared = prepare(follower); try (MockedConstruction executors = Mockito.mockConstruction(StmtExecutor.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/TableCacheInvalidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/TableCacheInvalidationTest.java new file mode 100644 index 00000000000000..da15a62e444b57 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/TableCacheInvalidationTest.java @@ -0,0 +1,617 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.View; +import org.apache.doris.catalog.constraint.UniqueConstraint; +import org.apache.doris.common.Config; +import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; +import org.apache.doris.common.cache.NereidsSqlCacheManager; +import org.apache.doris.common.io.Writable; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.journal.JournalEntity; +import org.apache.doris.mtmv.MTMVRelationManager; +import org.apache.doris.mtmv.MTMVService; +import org.apache.doris.nereids.SqlCacheContext; +import org.apache.doris.nereids.SqlCacheContext.FullTableName; +import org.apache.doris.nereids.SqlCacheContext.ScanTable; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.stats.SimpleAggCacheMgr; +import org.apache.doris.persist.AlterConstraintLog; +import org.apache.doris.persist.AlterViewInfo; +import org.apache.doris.persist.CreateTableInfo; +import org.apache.doris.persist.EditLog; +import org.apache.doris.persist.OperationType; +import org.apache.doris.persist.ReplaceTableOperationLog; +import org.apache.doris.persist.SetPartitionVersionOperationLog; +import org.apache.doris.persist.TableInfo; +import org.apache.doris.persist.TableRenameColumnInfo; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.cache.CacheAnalyzer; +import org.apache.doris.qe.cache.SqlCache; +import org.apache.doris.thrift.TUniqueId; +import org.apache.doris.transaction.PartitionCommitInfo; +import org.apache.doris.transaction.TableCommitInfo; +import org.apache.doris.transaction.TransactionState; +import org.apache.doris.transaction.TransactionStatus; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.gson.JsonObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +class TableCacheInvalidationTest extends TestWithFeService { + private final RecordingSimpleAggCache aggregates = new RecordingSimpleAggCache(); + private int sqlCapacity; + private int partitionCapacity; + private Database database; + private OlapTable target; + private OlapTable control; + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("cache_replay"); + } + + @BeforeEach + void prepareTables() throws Exception { + sqlCapacity = Config.sql_cache_manage_num; + partitionCapacity = Config.cache_partition_meta_table_manage_num; + Config.sql_cache_manage_num = 100; + Config.cache_partition_meta_table_manage_num = 100; + NereidsSqlCacheManager.updateConfig(); + NereidsSortedPartitionsCacheManager.updateConfig(); + connectContext.setDatabase("cache_replay"); + connectContext.getSessionVariable().setEnableSqlCache(true); + connectContext.getSessionVariable().cacheSortedPartitionIntervalSecond = -1; + dropTable("cache_replay.target", false); + dropTable("cache_replay.control", false); + dropTable("cache_replay.renamed", false); + dropTable("cache_replay.plain", false); + for (String name : ImmutableList.of("target", "control")) { + createTable("CREATE TABLE cache_replay." + name + " (id INT, v INT) DUPLICATE KEY(id) " + + "PARTITION BY RANGE(id) (PARTITION p1 VALUES LESS THAN (10), " + + "PARTITION p2 VALUES LESS THAN (20)) DISTRIBUTED BY HASH(id) BUCKETS 1 " + + "PROPERTIES ('replication_num'='1')"); + } + createTable("CREATE TABLE cache_replay.plain (id INT, v INT) DUPLICATE KEY(id) " + + "DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES ('replication_num'='1')"); + database = Env.getCurrentInternalCatalog().getDbOrMetaException("cache_replay"); + target = (OlapTable) database.getTableOrMetaException("target"); + control = (OlapTable) database.getTableOrMetaException("control"); + Env.getCurrentEnv().getSqlCacheManager().invalidateAll(); + Env.getCurrentEnv().getSortedPartitionsCacheManager().invalidateAll(); + aggregates.invalidated.clear(); + aggregates.onInvalidate = () -> { }; + SimpleAggCacheMgr.setTestInstance(aggregates); + } + + @AfterEach + void restoreAggregateCache() { + SimpleAggCacheMgr.clearTestInstance(); + Config.sql_cache_manage_num = sqlCapacity; + Config.cache_partition_meta_table_manage_num = partitionCapacity; + NereidsSqlCacheManager.updateConfig(); + NereidsSortedPartitionsCacheManager.updateConfig(); + } + + @ParameterizedTest + @CsvSource({"false", "true"}) + void constraintMutationInvalidatesCachesForLiveAndReplay(boolean replayJournal) throws Exception { + warmBothTables(); + UniqueConstraint constraint = new UniqueConstraint("cache_unique", ImmutableSet.of("id")); + if (replayJournal) { + replay(OperationType.OP_ADD_CONSTRAINT, + new AlterConstraintLog(constraint, target)); + } else { + target.addUniqueConstraint("cache_unique", ImmutableList.of("id"), false); + } + assertTargetInvalidated(); + warmBothTables(); + if (replayJournal) { + replay(OperationType.OP_DROP_CONSTRAINT, + new AlterConstraintLog(constraint, target)); + } else { + target.dropConstraint("cache_unique", false); + } + assertTargetInvalidated(); + } + + @ParameterizedTest + @CsvSource({"false", "true"}) + void removingPrimaryKeyInvalidatesTheReferencingTables(boolean dropPrimaryTable) throws Exception { + target.addPrimaryKeyConstraint("cache_pk", ImmutableList.of("id"), true); + control.addForeignConstraint("cache_fk", ImmutableList.of("id"), target, ImmutableList.of("id"), true); + warmBothTables(); + if (dropPrimaryTable) { + dropTable("cache_replay.target", false); + } else { + target.dropConstraint("cache_pk", true); + } + Assertions.assertFalse(control.getConstraintsMapUnsafe().containsKey("cache_fk")); + Assertions.assertEquals(0, Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().estimatedSize()); + Assertions.assertTrue(aggregates.invalidated.containsAll(ImmutableList.of(target.getId(), control.getId()))); + } + + @Test + void replayLegacyCreateTableInvalidatesAllTableCaches() throws Exception { + warmBothTables(); + JsonObject legacy = GsonUtils.GSON.toJsonTree( + new CreateTableInfo(database.getFullName(), database.getId(), target)).getAsJsonObject(); + for (String field : ImmutableList.of("ctl", "tbl", "dbId")) { + legacy.remove(field); + } + CreateTableInfo info = GsonUtils.GSON.fromJson(legacy, CreateTableInfo.class); + Assertions.assertNull(info.getCtlName()); + Assertions.assertNull(info.getTblName()); + Assertions.assertEquals(-1L, info.getDbId()); + database.unregisterTable(target.getId()); + + replay(OperationType.OP_CREATE_TABLE, info); + + Assertions.assertEquals(target.getId(), database.getTableOrMetaException("target").getId()); + Assertions.assertNotSame(target, database.getTableNullable("target")); + assertTargetInvalidated(); + } + + @Test + void replayRenameColumnInvalidatesAllTableCaches() throws Exception { + warmBothTables(); + long dataVersion = target.getVisibleVersion(); + TableRenameColumnInfo info = new TableRenameColumnInfo(database.getId(), target.getId(), "v", "new_v", + ImmutableMap.of(target.getBaseIndexId(), target.getBaseSchemaVersion() + 1)); + replay(OperationType.OP_RENAME_COLUMN, info); + Assertions.assertNull(target.getColumn("v")); + Assertions.assertNotNull(target.getColumn("new_v")); + Assertions.assertEquals(dataVersion, target.getVisibleVersion()); + assertTargetInvalidated(); + } + + @Test + void replayRenameTableRemovesSortedEntryUnderOldName() throws Exception { + warmBothTables(); + List namesAtNotification = new ArrayList<>(); + aggregates.onInvalidate = () -> namesAtNotification.add(target.getName()); + replay(OperationType.OP_RENAME_TABLE, + TableInfo.createForTableRename(database.getId(), target.getId(), "target", "renamed")); + Assertions.assertEquals("renamed", target.getName()); + Assertions.assertEquals(ImmutableList.of("renamed"), namesAtNotification); + assertTargetInvalidated(); + getSQLPlanOrErrorMsg("SELECT id FROM cache_replay.renamed WHERE id=1"); + Assertions.assertEquals(2, Env.getCurrentEnv().getSortedPartitionsCacheManager() + .getPartitionCaches().estimatedSize()); + } + + @ParameterizedTest + @CsvSource({"false", "true"}) + void replayReplaceTableInvalidatesBothIdentities(boolean swap) throws Exception { + warmBothTables(); + List identitiesAtNotification = new ArrayList<>(); + aggregates.onInvalidate = () -> identitiesAtNotification.add(database.getTableNullable("target").getId()); + replay(OperationType.OP_REPLACE_TABLE, new ReplaceTableOperationLog(database.getId(), + target.getId(), "target", control.getId(), "control", swap, false)); + Assertions.assertSame(control, database.getTableNullable("target")); + Assertions.assertFalse(identitiesAtNotification.isEmpty()); + Assertions.assertTrue(identitiesAtNotification.stream().allMatch(id -> id == control.getId())); + if (swap) { + Assertions.assertSame(target, database.getTableNullable("control")); + } else { + Assertions.assertNull(database.getTableNullable("control")); + } + Assertions.assertEquals(0, Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().estimatedSize()); + Assertions.assertEquals(0, Env.getCurrentEnv().getSortedPartitionsCacheManager() + .getPartitionCaches().estimatedSize()); + Assertions.assertTrue(aggregates.invalidated.containsAll(ImmutableList.of(target.getId(), control.getId()))); + } + + @Test + void replayViewDefinitionInvalidatesQueriesUsingTheView() throws Exception { + executeNereidsSql("DROP VIEW IF EXISTS cache_replay.cached_view"); + createView("CREATE VIEW cache_replay.cached_view AS SELECT id FROM cache_replay.target"); + View view = (View) database.getTableOrMetaException("cached_view"); + SqlCacheContext context = warmSql("view", target); + context.addUsedView(view, view.getInlineViewDef()); + warmSql("control", control); + replay(OperationType.OP_MODIFY_VIEW_DEF, new AlterViewInfo(database.getId(), view.getId(), + "SELECT v FROM cache_replay.target", view.getFullSchema(), ImmutableMap.of(), "")); + Assertions.assertNull(Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().getIfPresent("view")); + Assertions.assertNotNull(Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().getIfPresent("control")); + } + + @Test + void visibleReplayDoesNotScanCachesAndValidatesDependenciesOnLookup() throws Exception { + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + String first = "SELECT id FROM target PARTITION(p1)"; + String second = "SELECT id FROM target PARTITION(p2)"; + String whole = "SELECT id FROM target"; + String other = "SELECT id FROM control PARTITION(p1)"; + prepareCachedQuery(first, target); + publishCachedQuery(false, first); + prepareCachedQuery(second, target, "p2"); + publishCachedQuery(false, second); + prepareCachedQuery(whole, target, null); + publishCachedQuery(false, whole); + prepareCachedQuery(other, control); + publishCachedQuery(false, other); + Assertions.assertEquals(4, manager.getSqlCaches().estimatedSize()); + manager.getSqlCaches().asMap().replaceAll((key, cached) -> Mockito.spy(cached)); + List cachedContexts = new ArrayList<>(manager.getSqlCaches().asMap().values()); + + TransactionState transaction = committedTransaction(target, "p2"); + transaction.setTransactionStatus(TransactionStatus.VISIBLE); + replay(OperationType.OP_UPSERT_TRANSACTION_STATE, transaction); + + for (SqlCacheContext cached : cachedContexts) { + Mockito.verify(cached, Mockito.never()).getUsedTables(); + } + Assertions.assertEquals(4, manager.getSqlCaches().estimatedSize(), + "VISIBLE must publish versions without walking or evicting cached queries under table write locks"); + assertCacheHit(first, false); + assertCacheHit(second, false); + assertCacheHit(whole, false); + assertCacheHit(other, true); + Assertions.assertEquals(1, manager.getSqlCaches().estimatedSize()); + Assertions.assertTrue(aggregates.invalidated.isEmpty()); + } + + @ParameterizedTest + @CsvSource({"false", "true"}) + void transactionReplayChecksDataWhenVisibleButNotWhenCommitted(boolean partitioned) throws Exception { + OlapTable table = partitioned ? target : (OlapTable) database.getTableOrMetaException("plain"); + String partitionName = partitioned ? "p1" : table.getName(); + String query = "SELECT id FROM " + table.getName(); + prepareCachedQuery(query, table); + publishCachedQuery(false, query); + long oldVersion = table.getVisibleVersion(); + TransactionState transaction = committedTransaction(table, partitionName); + replay(OperationType.OP_UPSERT_TRANSACTION_STATE, transaction); + Assertions.assertEquals(oldVersion, table.getVisibleVersion()); + assertCacheHit(query, true); + + transaction.setTransactionStatus(TransactionStatus.VISIBLE); + replay(OperationType.OP_UPSERT_TRANSACTION_STATE, transaction); + Assertions.assertEquals(oldVersion + 1, table.getVisibleVersion()); + Assertions.assertEquals(1, Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().estimatedSize()); + assertCacheHit(query, false); + Assertions.assertEquals(0, Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().estimatedSize()); + Assertions.assertTrue(aggregates.invalidated.isEmpty()); + } + + private TransactionState committedTransaction(OlapTable table, String partitionName) throws Exception { + long now = System.currentTimeMillis(); + TransactionState transaction = new TransactionState(database.getId(), ImmutableList.of(table.getId()), + Env.getCurrentEnv().getNextId(), "cache_replay_visible", null, + TransactionState.LoadJobSourceType.FRONTEND, + new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.FE, 0, "127.0.0.1", now), + -1, 10000); + TableCommitInfo commit = new TableCommitInfo(table.getId()); + commit.setVersion(table.getVisibleVersion() + 1); + commit.setVersionTime(now); + Partition partition = table.getPartition(partitionName); + commit.addPartitionCommitInfo(new PartitionCommitInfo(partition.getId(), "", + partition.getVisibleVersion() + 1, now, false)); + transaction.putIdToTableCommitInfo(table.getId(), commit); + transaction.setCommitTime(now); + transaction.setFinishTime(now); + transaction.setTransactionStatus(TransactionStatus.COMMITTED); + return transaction; + } + + private void assertCacheHit(String query, boolean expected) { + try (StatementContext lookup = new StatementContext(connectContext, new OriginStatement(query, 0))) { + connectContext.setStatementContext(lookup); + Assertions.assertEquals(expected, + Env.getCurrentEnv().getSqlCacheManager().tryParseSql(connectContext, query).isPresent(), query); + } + } + + @Test + void nonPartitionedCacheChecksSchemaAndChangesBackendKey() throws Exception { + OlapTable table = (OlapTable) database.getTableOrMetaException("plain"); + String query = "SELECT * FROM plain"; + SqlCacheContext cached = prepareCachedQuery(query, table); + Env env = Env.getCurrentEnv(); + env.getSqlCacheManager().tryAddFeSqlCache(connectContext, query); + String oldKey = cached.getUsedTables().values().iterator().next().getCacheKey(); + try (StatementContext lookup = new StatementContext(connectContext, new OriginStatement(query, 0))) { + connectContext.setStatementContext(lookup); + Assertions.assertTrue(env.getSqlCacheManager().tryParseSql(connectContext, query).isPresent()); + } + // Model a plan published after the eager invalidation has already run. + table.getIndexMetaByIndexId(table.getBaseIndexId()).setSchemaVersion(table.getBaseSchemaVersion() + 1); + SqlCacheContext current = new SqlCacheContext(UserIdentity.ROOT); + current.addUsedTable(table); + Assertions.assertNotEquals(oldKey, current.getUsedTables().values().iterator().next().getCacheKey()); + try (StatementContext lookup = new StatementContext(connectContext, new OriginStatement(query, 0))) { + connectContext.setStatementContext(lookup); + Assertions.assertFalse(env.getSqlCacheManager().tryParseSql(connectContext, query).isPresent()); + } + Assertions.assertEquals(0, env.getSqlCacheManager().getSqlCaches().estimatedSize()); + } + + @ParameterizedTest + @CsvSource({"false,false", "false,true", "true,false", "true,true"}) + void metadataReplayRejectsInFlightPublication(boolean rename, boolean backend) throws Exception { + target.getPartition("p1").updateVisibleVersionAndTime(3L, 123456L); + String query = "SELECT id FROM target PARTITION(p1)"; + SqlCacheContext cached = prepareCachedQuery(query, target); + StatementContext statement = connectContext.getStatementContext(); + SqlCacheContext unrelated = prepareCachedQuery("SELECT id FROM control", control); + connectContext.setStatementContext(statement); + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + long tableVersion = target.getVisibleVersion(); + int schemaVersion = target.getBaseSchemaVersion(); + + replayPartitionMetadataChange(rename); + + Assertions.assertEquals(tableVersion, target.getVisibleVersion()); + Assertions.assertEquals(schemaVersion, target.getBaseSchemaVersion()); + publishCachedQuery(backend, query); + Assertions.assertEquals(0, manager.getSqlCaches().estimatedSize(), + "an executing query must not republish its old metadata dependencies"); + Assertions.assertTrue(cached.supportSqlCache()); + Assertions.assertFalse(cached.metadataVersionsMatch()); + Assertions.assertTrue(unrelated.metadataVersionsMatch()); + } + + @ParameterizedTest + @CsvSource({"false", "true"}) + void metadataReplayRejectsContextPutAfterEviction(boolean rename) throws Exception { + target.getPartition("p1").updateVisibleVersionAndTime(3L, 123456L); + String query = "SELECT id FROM target PARTITION(p1)"; + SqlCacheContext cached = prepareCachedQuery(query, target); + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + manager.tryAddFeSqlCache(connectContext, query); + String key = manager.getSqlCaches().asMap().keySet().iterator().next(); + + replayPartitionMetadataChange(rename); + Assertions.assertEquals(0, manager.getSqlCaches().estimatedSize()); + // Model invalidation between the publisher's final check and its put. + manager.getSqlCaches().put(key, cached); + + try (StatementContext lookup = new StatementContext(connectContext, new OriginStatement(query, 0))) { + connectContext.setStatementContext(lookup); + Assertions.assertFalse(manager.tryParseSql(connectContext, query).isPresent(), + "a late put must not make an invalidated context usable again"); + } + Assertions.assertEquals(0, manager.getSqlCaches().estimatedSize()); + } + + @ParameterizedTest + @CsvSource({"false", "true"}) + void publicationValidatesDataWithoutAnInvalidationNotification(boolean backend) throws Exception { + target.getPartition("p1").updateVisibleVersionAndTime(3L, 123456L); + String query = "SELECT id FROM target PARTITION(p1)"; + SqlCacheContext cached = prepareCachedQuery(query, target); + target.getPartition("p1").updateVisibleVersionAndTime(4L, 123457L); + target.updateVisibleVersionAndTime(target.getVisibleVersion() + 1, 123457L); + Assertions.assertTrue(cached.metadataVersionsMatch()); + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + publishCachedQuery(backend, query); + Assertions.assertEquals(0, manager.getSqlCaches().estimatedSize()); + } + + @ParameterizedTest + @CsvSource({"false,false", "false,true", "true,false", "true,true"}) + void publicationPreservesBranchTableVersionValidation(boolean backend, boolean writeOtherPartition) + throws Exception { + String query = "SELECT id FROM target PARTITION(p1)"; + SqlCacheContext cached = prepareCachedQuery(query, target); + if (writeOtherPartition) { + target.writeLock(); + try { + Partition p2 = target.getPartition("p2"); + p2.updateVisibleVersionAndTime(p2.getVisibleVersion() + 1, 123457L); + target.updateVisibleVersionAndTime(target.getVisibleVersion() + 1, 123457L); + } finally { + target.writeUnlock(); + } + } + publishCachedQuery(backend, query); + Assertions.assertTrue(cached.metadataVersionsMatch()); + Assertions.assertEquals(writeOtherPartition ? 0 : 1, + Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().estimatedSize()); + } + + private void publishCachedQuery(boolean backend, String query) { + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + if (backend) { + CacheAnalyzer analyzer = Mockito.mock(CacheAnalyzer.class); + Mockito.when(analyzer.getCache()).thenReturn(Mockito.mock(SqlCache.class)); + Mockito.when(analyzer.getScanTables()).thenReturn(ImmutableList.of()); + manager.tryAddBeCache(connectContext, query, analyzer); + } else { + manager.tryAddFeSqlCache(connectContext, query); + } + } + + private void replayPartitionMetadataChange(boolean rename) throws Exception { + Partition partition = target.getPartition("p1"); + if (rename) { + replay(OperationType.OP_RENAME_PARTITION, TableInfo.createForPartitionRename( + database.getId(), target.getId(), partition.getId(), "p1", "renamed_partition")); + } else { + replay(OperationType.OP_SET_PARTITION_VERSION, new SetPartitionVersionOperationLog( + database.getFullName(), target.getName(), partition.getId(), 2L)); + } + } + + @ParameterizedTest + @CsvSource({"database", "catalog", "all"}) + void namespaceInvalidationReachesExecutingQueries(String scope) { + SqlCacheContext cached = prepareCachedQuery("SELECT id FROM target", target); + StatementContext statement = connectContext.getStatementContext(); + // A constant statement has no table dependencies. + StatementContext constantStatement = new StatementContext(connectContext, new OriginStatement("SELECT 1", 0)); + SqlCacheContext constant = constantStatement.getSqlCacheContext().get(); + connectContext.setStatementContext(statement); + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + // Replacing the result-cache container must preserve dependency revisions. + NereidsSqlCacheManager.updateConfig(); + switch (scope) { + case "database": + Env.getCurrentEnv().onDatabaseChanged(InternalCatalog.INTERNAL_CATALOG_NAME, "cache_replay"); + break; + case "catalog": + Env.getCurrentEnv().onCatalogChanged(InternalCatalog.INTERNAL_CATALOG_NAME); + break; + case "all": + manager.invalidateAll(); + break; + default: + throw new AssertionError(scope); + } + manager.tryAddFeSqlCache(connectContext, "SELECT id FROM target"); + Assertions.assertEquals(0, manager.getSqlCaches().estimatedSize()); + Assertions.assertFalse(cached.metadataVersionsMatch()); + Assertions.assertEquals(!scope.equals("all"), constant.metadataVersionsMatch()); + } + + private SqlCacheContext prepareCachedQuery(String query, OlapTable table) { + return prepareCachedQuery(query, table, "p1"); + } + + private SqlCacheContext prepareCachedQuery(String query, OlapTable table, String partitionName) { + StatementContext statement = new StatementContext(connectContext, new OriginStatement(query, 0)); + connectContext.setStatementContext(statement); + SqlCacheContext cached = statement.getSqlCacheContext().get(); + cached.addUsedTable(table); + if (table.isPartitionedTable() && partitionName != null) { + Partition partition = table.getPartition(partitionName); + ScanTable scan = new ScanTable(new FullTableName( + InternalCatalog.INTERNAL_CATALOG_NAME, database.getFullName(), table.getName())); + scan.addScanPartition(partition.getId()); + cached.addScanTable(scan, table); + } + cached.setPhysicalPlan("cached plan"); + cached.setColLabels(ImmutableList.of("id", "v")); + cached.setFieldInfos(ImmutableList.of()); + cached.setResultExprs(ImmutableList.of()); + cached.setQueryId(new TUniqueId(1, 1)); + cached.setResultSetInFe(new CommonResultSet( + new CommonResultSet.CommonResultSetMetaData(ImmutableList.of()), ImmutableList.of())); + connectContext.setQueryId(new TUniqueId(1, 1)); + return cached; + } + + @Test + void databaseInvalidationPreservesOtherNamespaces() { + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + warmSql("target", target); + SqlCacheContext other = new SqlCacheContext(UserIdentity.ROOT); + manager.getSqlCaches().put("constant", other); + Env.getCurrentEnv().onDatabaseChanged(InternalCatalog.INTERNAL_CATALOG_NAME, "cache_replay"); + Assertions.assertNull(manager.getSqlCaches().getIfPresent("target")); + Assertions.assertNotNull(manager.getSqlCaches().getIfPresent("constant")); + } + + @Test + void invalidationFailuresAreIsolatedPerCache() { + NereidsSqlCacheManager sql = Mockito.mock(NereidsSqlCacheManager.class); + NereidsSortedPartitionsCacheManager sorted = Mockito.mock(NereidsSortedPartitionsCacheManager.class); + MTMVService mtmv = Mockito.mock(MTMVService.class); + MTMVRelationManager relations = Mockito.mock(MTMVRelationManager.class); + Env env = Mockito.mock(Env.class, Mockito.CALLS_REAL_METHODS); + Mockito.when(env.getMtmvService()).thenReturn(mtmv); + Mockito.when(mtmv.getRelationManager()).thenReturn(relations); + Mockito.when(env.getSqlCacheManager()).thenReturn(sql); + Mockito.when(env.getSortedPartitionsCacheManager()).thenReturn(sorted); + Mockito.doThrow(new IllegalStateException("SQL invalidation failed")) + .when(sql).invalidateAboutTable(Mockito.eq(target.getId()), Mockito.any(FullTableName.class)); + Mockito.doThrow(new IllegalStateException("partition invalidation failed")) + .when(sorted).invalidateTable(target.getId()); + Mockito.doThrow(new AssertionError("MV invalidation failed")).when(relations) + .invalidateRewriteCaches(InternalCatalog.INTERNAL_CATALOG_NAME, "cache_replay", "target"); + Assertions.assertDoesNotThrow(() -> env.onTableChanged(target)); + Mockito.verify(sorted).invalidateTable(target.getId()); + Mockito.verify(relations).invalidateRewriteCaches( + InternalCatalog.INTERNAL_CATALOG_NAME, "cache_replay", "target"); + Assertions.assertTrue(aggregates.invalidated.contains(target.getId())); + } + + private SqlCacheContext warmSql(String key, OlapTable table) { + SqlCacheContext context = new SqlCacheContext(UserIdentity.ROOT, + Env.getCurrentEnv().getSqlCacheManager().newMetadataSnapshot()); + context.addUsedTable(table); + Env.getCurrentEnv().getSqlCacheManager().getSqlCaches().put(key, context); + return context; + } + + private void warmBothTables() throws Exception { + getSQLPlanOrErrorMsg("SELECT id FROM cache_replay.target WHERE id=1"); + getSQLPlanOrErrorMsg("SELECT id FROM cache_replay.control WHERE id=1"); + warmSql("target", target); + warmSql("control", control); + Assertions.assertEquals(2, Env.getCurrentEnv().getSortedPartitionsCacheManager() + .getPartitionCaches().estimatedSize()); + } + + private void assertTargetInvalidated() { + NereidsSqlCacheManager manager = Env.getCurrentEnv().getSqlCacheManager(); + Assertions.assertNull(manager.getSqlCaches().getIfPresent("target")); + Assertions.assertNotNull(manager.getSqlCaches().getIfPresent("control")); + Assertions.assertEquals(1, Env.getCurrentEnv().getSortedPartitionsCacheManager() + .getPartitionCaches().estimatedSize()); + Assertions.assertTrue(aggregates.invalidated.contains(target.getId())); + Assertions.assertFalse(aggregates.invalidated.contains(control.getId())); + } + + private void replay(short operation, Writable payload) throws Exception { + JournalEntity written = new JournalEntity(); + written.setOpCode(operation); + written.setData(payload); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + written.write(new DataOutputStream(bytes)); + JournalEntity read = new JournalEntity(); + read.readFields(new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))); + EditLog.loadJournal(Env.getCurrentEnv(), 1L, read); + } + + private static class RecordingSimpleAggCache extends SimpleAggCacheMgr { + private final Set invalidated = new HashSet<>(); + private Runnable onInvalidate = () -> { }; + + @Override + public void invalidateTable(long tableId) { + invalidated.add(tableId); + onInvalidate.run(); + } + } +} diff --git a/regression-test/data/nereids_p0/cache/sql_cache_metadata_replay.out b/regression-test/data/nereids_p0/cache/sql_cache_metadata_replay.out new file mode 100644 index 00000000000000..7bdf7d92f875ed --- /dev/null +++ b/regression-test/data/nereids_p0/cache/sql_cache_metadata_replay.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_drop -- +1 11 99 + +-- !after_drop -- +1 11 + +-- !before_rename -- +11 + +-- !after_rename -- +1 11 + +-- !after_insert -- +1 11 +2 22 + +-- !view_before -- +1 11 +2 22 + +-- !view_after -- +1 111 +2 122 + +-- !after_truncate -- + diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy index ae4437075ec158..5f3a25756a1b2e 100644 --- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy @@ -103,7 +103,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select * from test_use_plan_cache" // create sql cache - sql "select * from test_use_plan_cache" + retryUntilHasSqlCache "select * from test_use_plan_cache" // use sql cache assertHasCache "select * from test_use_plan_cache" @@ -122,7 +122,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache2" - sql "select * from test_use_plan_cache2" + retryUntilHasSqlCache "select * from test_use_plan_cache2" assertHasCache "select * from test_use_plan_cache2" // NOTE: in cloud mode, add empty partition can not use cache, because the table version already update, @@ -149,14 +149,14 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from ${tb_name}" - sql "select * from ${tb_name}" + retryUntilHasSqlCache "select * from ${tb_name}" assertHasCache "select * from ${tb_name}" // insert overwrite data can not use cache sql "INSERT OVERWRITE table ${tb_name} PARTITION(p5) VALUES (5, 6);" sleep(10 * 1000) assertNoCache "select * from ${tb_name}" - sql "select * from ${tb_name}" + retryUntilHasSqlCache "select * from ${tb_name}" assertHasCache "select * from ${tb_name}" sql "alter table ${tb_name} add partition p6 values[('6'),('7'))" @@ -180,7 +180,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache3" - sql "select * from test_use_plan_cache3" + retryUntilHasSqlCache "select * from test_use_plan_cache3" assertHasCache "select * from test_use_plan_cache3" // drop partition can not use cache @@ -210,7 +210,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache4" - sql "select * from test_use_plan_cache4" + retryUntilHasSqlCache "select * from test_use_plan_cache4" assertHasCache "select * from test_use_plan_cache4" // replace partition can not use cache @@ -231,7 +231,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache5" - sql "select * from test_use_plan_cache5" + retryUntilHasSqlCache "select * from test_use_plan_cache5" assertHasCache "select * from test_use_plan_cache5" streamLoad { @@ -258,7 +258,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache6" - sql "select * from test_use_plan_cache6" + retryUntilHasSqlCache "select * from test_use_plan_cache6" assertHasCache "select * from test_use_plan_cache6" sql "update test_use_plan_cache6 set value=3 where id=1" @@ -280,7 +280,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache7" - sql "select * from test_use_plan_cache7" + retryUntilHasSqlCache "select * from test_use_plan_cache7" assertHasCache "select * from test_use_plan_cache7" sql "delete from test_use_plan_cache7 where id = 1" @@ -302,7 +302,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache8" - sql "select * from test_use_plan_cache8" + retryUntilHasSqlCache "select * from test_use_plan_cache8" assertHasCache "select * from test_use_plan_cache8" sql "drop table test_use_plan_cache8" @@ -330,7 +330,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache9_view" - sql "select * from test_use_plan_cache9_view" + retryUntilHasSqlCache "select * from test_use_plan_cache9_view" assertHasCache "select * from test_use_plan_cache9_view" // alter view should not use cache @@ -354,7 +354,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache10_view" - sql "select * from test_use_plan_cache10_view" + retryUntilHasSqlCache "select * from test_use_plan_cache10_view" assertHasCache "select * from test_use_plan_cache10_view" sql "drop view test_use_plan_cache10_view" @@ -381,7 +381,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache11_view" - sql "select * from test_use_plan_cache11_view" + retryUntilHasSqlCache "select * from test_use_plan_cache11_view" assertHasCache "select * from test_use_plan_cache11_view" sql "insert into test_use_plan_cache11 values(1, 3)" @@ -414,7 +414,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache12" - sql "select * from test_use_plan_cache12" + retryUntilHasSqlCache "select * from test_use_plan_cache12" assertHasCache "select * from test_use_plan_cache12" sql "sync" @@ -467,7 +467,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache13" - sql "select * from test_use_plan_cache13" + retryUntilHasSqlCache "select * from test_use_plan_cache13" assertHasCache "select * from test_use_plan_cache13" } }).get() @@ -534,7 +534,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache14" - sql "select * from test_use_plan_cache14" + retryUntilHasSqlCache "select * from test_use_plan_cache14" assertHasCache "select * from test_use_plan_cache14" } }).get() @@ -591,7 +591,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache15" - sql "select * from test_use_plan_cache15" + retryUntilHasSqlCache "select * from test_use_plan_cache15" assertHasCache "select * from test_use_plan_cache15" } }).get() @@ -635,7 +635,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select random() from test_use_plan_cache16" assertNoCache "select year(now()) from test_use_plan_cache16" - sql "select year(now()) from test_use_plan_cache16" + retryUntilHasSqlCache "select year(now()) from test_use_plan_cache16" assertHasCache "select year(now()) from test_use_plan_cache16" assertNoCache "select second(now()) from test_use_plan_cache16" @@ -660,7 +660,7 @@ suite("parse_sql_from_sql_cache") { sql "set @custom_variable=10" assertNoCache "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" // create sql cache - sql "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" + retryUntilHasSqlCache "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" // can use sql cache assertHasCache "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" @@ -668,6 +668,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" def result2 = sql "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" + retryUntilHasSqlCache "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" assertHasCache "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1" assertTrue(result2.size() == 1 && result2[0][0].toString().toInteger() == 20) @@ -684,6 +685,7 @@ suite("parse_sql_from_sql_cache") { def functionResult = sql functionVariableSql assertTrue(functionResult.size() == 1 && functionResult[0][0].toString().toInteger() == 10) + retryUntilHasSqlCache functionVariableSql assertHasCache functionVariableSql sql "set @custom_variable_in_function=-20" @@ -691,6 +693,7 @@ suite("parse_sql_from_sql_cache") { functionResult = sql functionVariableSql assertTrue(functionResult.size() == 1 && functionResult[0][0].toString().toInteger() == 20) + retryUntilHasSqlCache functionVariableSql assertHasCache functionVariableSql // switch back to the original value and reuse its value-aware cache @@ -704,6 +707,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" def res = sql "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" assertTrue(res[0][0] == 1) + retryUntilHasSqlCache "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" assertHasCache "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" sql "set @custom_variable2=2" @@ -711,6 +715,7 @@ suite("parse_sql_from_sql_cache") { // should not invalidate cache with @custom_variable2=1 res = sql "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" assertTrue(res[0][0] == 2) + retryUntilHasSqlCache "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" assertHasCache "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1" sql "set @custom_variable2=1" @@ -805,7 +810,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select * from test_use_plan_cache18" - sql "select * from test_use_plan_cache18" + retryUntilHasSqlCache "select * from test_use_plan_cache18" assertHasCache "select * from test_use_plan_cache18" } @@ -850,6 +855,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select * from test_use_plan_cache19 order by 1, 2" def result2 = sql "select * from test_use_plan_cache19 order by 1, 2" assertTrue(result2.size() > 1) + retryUntilHasSqlCache "select * from test_use_plan_cache19 order by 1, 2" assertHasCache "select * from test_use_plan_cache19 order by 1, 2" sql "set dry_run_query=true" @@ -879,11 +885,12 @@ suite("parse_sql_from_sql_cache") { def result1 = sql "select * from (select $randomInt as id)a" assertTrue(result1.size() == 1) + retryUntilHasSqlCache "select * from (select $randomInt as id)a" assertHasCache "select * from (select $randomInt as id)a" def result2 = sql "select * from (select $randomInt as id)a" assertTrue(result2.size() == 1) - sql "select * from test_use_plan_cache20 limit 0" + retryUntilHasSqlCache "select * from test_use_plan_cache20 limit 0" assertHasCache "select * from test_use_plan_cache20 limit 0" def result4 = sql "select * from test_use_plan_cache20 limit 0" assertTrue(result4.isEmpty()) @@ -891,6 +898,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select * from test_use_plan_cache20 where id=999" def result5 = sql "select * from test_use_plan_cache20 where id=999" assertTrue(result5.isEmpty()) + retryUntilHasSqlCache "select * from test_use_plan_cache20 where id=999" assertHasCache "select * from test_use_plan_cache20 where id=999" def result6 = sql "select * from test_use_plan_cache20 where id=999" assertTrue(result6.isEmpty()) @@ -926,6 +934,7 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select * from test_use_plan_cache21" def result1 = sql "select * from test_use_plan_cache21" assertTrue(result1.size() == 2) + retryUntilHasSqlCache "select * from test_use_plan_cache21" assertHasCache "select * from test_use_plan_cache21" sql "truncate table test_use_plan_cache21 partition dt2" @@ -947,7 +956,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" assertNoCache "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment2*/ * from test_use_plan_cache22 order by 1, 2" - sql "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment1*/ * from test_use_plan_cache22 order by 1, 2" + retryUntilHasSqlCache "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment1*/ * from test_use_plan_cache22 order by 1, 2" assertHasCache "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment2*/ * from test_use_plan_cache22 order by 1, 2" } @@ -965,7 +974,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_strong_consistency_read=true" int randomInt = Math.random() * 2000000000 - sql "select ${randomInt} from test_use_plan_cache23" + retryUntilHasSqlCache "select ${randomInt} from test_use_plan_cache23" profile("sql_cache_23_${randomInt}") { run { sql "/* sql_cache_23_${randomInt} */ select ${randomInt} from test_use_plan_cache23" @@ -978,7 +987,7 @@ suite("parse_sql_from_sql_cache") { } randomInt = Math.random() * 2000000000 - sql "select * from (select $randomInt as id)a" + retryUntilHasSqlCache "select * from (select $randomInt as id)a" profile("sql_cache_23_${randomInt}_2") { run { sql "/* sql_cache_23_${randomInt}_2 */ select * from (select $randomInt as id)a" @@ -1086,6 +1095,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" assertTrue((sql "select * from test_use_plan_cache24").size() > 0) + retryUntilHasSqlCache "select * from test_use_plan_cache24" assertHasCache "select * from test_use_plan_cache24" connectToDoris(context.config.jdbcUser, context.config.jdbcPassword, context.jdbcUrl) { @@ -1114,7 +1124,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select count(1) from test_use_plan_cache25 group by id order by id" + retryUntilHasSqlCache "select count(1) from test_use_plan_cache25 group by id order by id" assertHasCache "select count(1) from test_use_plan_cache25 group by id order by id" sql "alter table test_use_plan_cache25 rename column id id2" @@ -1135,7 +1145,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select count(1) from test_use_plan_cache26 group by id order by id" + retryUntilHasSqlCache "select count(1) from test_use_plan_cache26 group by id order by id" assertHasCache "select count(1) from test_use_plan_cache26 group by id order by id" sql "alter table test_use_plan_cache26 rename test_use_plan_cache26_bak" @@ -1157,7 +1167,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select count(1) from test_use_plan_cache27 group by id order by id" + retryUntilHasSqlCache "select count(1) from test_use_plan_cache27 group by id order by id" assertHasCache "select count(1) from test_use_plan_cache27 group by id order by id" sql "alter table test_use_plan_cache27 replace with table test_use_plan_cache27_bak" @@ -1174,7 +1184,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select count(1) from test_use_plan_cache28 group by id order by id" + retryUntilHasSqlCache "select count(1) from test_use_plan_cache28 group by id order by id" assertHasCache "select count(1) from test_use_plan_cache28 group by id order by id" sql "truncate table test_use_plan_cache28" @@ -1191,7 +1201,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select * from test_use_plan_cache29" + retryUntilHasSqlCache "select * from test_use_plan_cache29" assertHasCache "select * from test_use_plan_cache29" sql "alter table test_use_plan_cache29 add column bbb int default '0'" @@ -1209,7 +1219,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select * from test_use_plan_cache30" + retryUntilHasSqlCache "select * from test_use_plan_cache30" assertHasCache "select * from test_use_plan_cache30" sql "alter table test_use_plan_cache30 drop column value" @@ -1226,7 +1236,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select * from test_use_plan_cache31" + retryUntilHasSqlCache "select * from test_use_plan_cache31" assertHasCache "select * from test_use_plan_cache31" sql "alter table test_use_plan_cache31 modify column value bigint" @@ -1245,7 +1255,7 @@ suite("parse_sql_from_sql_cache") { sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" - sql "select * from test_use_plan_cache32_view" + retryUntilHasSqlCache "select * from test_use_plan_cache32_view" assertHasCache "select * from test_use_plan_cache32_view" sql "insert into test_use_plan_cache32 values (1, 1)" diff --git a/regression-test/suites/nereids_p0/cache/sql_cache_metadata_replay.groovy b/regression-test/suites/nereids_p0/cache/sql_cache_metadata_replay.groovy new file mode 100644 index 00000000000000..294bd78d1cf48d --- /dev/null +++ b/regression-test/suites/nereids_p0/cache/sql_cache_metadata_replay.groovy @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("sql_cache_metadata_replay") { + withGlobalLock("cache_last_version_interval_second") { + def follower = sql_return_maparray("SHOW FRONTENDS").find { + it.Alive.toString().equalsIgnoreCase("true") && it.Join.toString().equalsIgnoreCase("true") + && !it.IsMaster.toString().equalsIgnoreCase("true") + } + if (follower == null) { + logger.info("SQL cache metadata replay requires a follower or observer FE") + return + } + def database = (sql "SELECT DATABASE()")[0][0] + def url = "jdbc:mysql://${follower.Host}:${follower.QueryPort}/${database}?useLocalSessionState=true" + connect(context.config.jdbcUser, context.config.jdbcPassword, url) { + def originalInterval = (sql "ADMIN SHOW FRONTEND CONFIG LIKE 'cache_last_version_interval_second'")[0][1] + sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second'='0')" + try { + sql "SET enable_sql_cache=true" + sql "SET enable_strong_consistency_read=true" + sql "DROP VIEW IF EXISTS sql_cache_replay_view" + sql "DROP TABLE IF EXISTS sql_cache_replay_table" + sql """CREATE TABLE sql_cache_replay_table (id INT, value INT, added INT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ('replication_num'='1', 'light_schema_change'='false')""" + sql "INSERT INTO sql_cache_replay_table VALUES (1,11,99)" + order_qt_before_drop "SELECT * FROM sql_cache_replay_table ORDER BY id" + retryUntilHasSqlCache("SELECT * FROM sql_cache_replay_table ORDER BY id") + explain { + sql "PHYSICAL PLAN SELECT * FROM sql_cache_replay_table ORDER BY id" + contains "PhysicalSqlCache" + } + sql "ALTER TABLE sql_cache_replay_table DROP COLUMN added" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE TableName='sql_cache_replay_table' + ORDER BY JobId DESC LIMIT 1""" + time 120 + } + order_qt_after_drop "SELECT * FROM sql_cache_replay_table ORDER BY id" + sql "ALTER TABLE sql_cache_replay_table SET ('light_schema_change'='true')" + sql "SYNC" + order_qt_before_rename "SELECT value FROM sql_cache_replay_table ORDER BY id" + retryUntilHasSqlCache("SELECT value FROM sql_cache_replay_table ORDER BY id") + explain { + sql "PHYSICAL PLAN SELECT value FROM sql_cache_replay_table ORDER BY id" + contains "PhysicalSqlCache" + } + sql "ALTER TABLE sql_cache_replay_table RENAME COLUMN value renamed_value" + sql "SYNC" + test { + sql "SELECT value FROM sql_cache_replay_table ORDER BY id" + exception "Unknown column" + } + order_qt_after_rename "SELECT id,renamed_value FROM sql_cache_replay_table ORDER BY id" + sql "INSERT INTO sql_cache_replay_table VALUES (2,22)" + order_qt_after_insert "SELECT id,renamed_value FROM sql_cache_replay_table ORDER BY id" + sql "CREATE VIEW sql_cache_replay_view AS SELECT id,renamed_value FROM sql_cache_replay_table" + order_qt_view_before "SELECT * FROM sql_cache_replay_view ORDER BY id" + sql """ALTER VIEW sql_cache_replay_view AS + SELECT id,renamed_value+100 AS renamed_value FROM sql_cache_replay_table""" + sql "SYNC" + order_qt_view_after "SELECT * FROM sql_cache_replay_view ORDER BY id" + sql "TRUNCATE TABLE sql_cache_replay_table" + sql "SYNC" + order_qt_after_truncate "SELECT * FROM sql_cache_replay_view ORDER BY id" + } finally { + sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second'='${originalInterval}')" + } + } + } +} From 44e6c2e79ec0b3dee84c9dc12d8cd7a123b31a8c Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 12 Nov 2026 15:54:20 +0800 Subject: [PATCH 2/4] [fix](fe) Preserve MV rewrite invalidation dependencies ### What problem does this PR solve? Related PR: #70397 Related PR: #70398 Problem Summary: External refresh can miss dependent MV rewrite caches when SQL name normalization changes a canonical mixed-case table name. A concurrent MV refresh can also temporarily remove the MV from its dependency indexes, allowing invalidation to miss a newly published pre-refresh plan. Build lookup identities from canonical metadata names and coordinate index replacement/unregistration with invalidation discovery. Release the index monitor before table resolution or MV cache invalidation to preserve lock order. ### Release note Fix missed MV rewrite-cache invalidation after external metadata refreshes. ### Check List (For Author) - Test: Unit Test: 72 focused FE tests passed. The new 9-case regression suite reproduced 7 failures before the fix; all cases pass after the fix. FE build and Checkstyle passed; multi-FE/BE regression was not run locally. - Behavior changed: Yes, preserve canonical dependency identity and avoid missing registered MVs during concurrent dependency replacement. - Does this need documentation: No --- .../org/apache/doris/mtmv/BaseTableInfo.java | 16 +- .../doris/mtmv/MTMVRelationManager.java | 58 ++-- .../mtmv/MTMVRewriteInvalidationRaceTest.java | 250 ++++++++++++++++++ 3 files changed, 296 insertions(+), 28 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteInvalidationRaceTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java index 3efe6cb225e448..adb4bf84322ddb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java @@ -75,13 +75,15 @@ public BaseTableInfo(TableIf table) { } public BaseTableInfo(TableNameInfo tableNameInfo) { - java.util.Objects.requireNonNull(tableNameInfo, "tableNameInfo is null"); - java.util.Objects.requireNonNull(tableNameInfo.getTbl(), "tableName is null"); - java.util.Objects.requireNonNull(tableNameInfo.getDb(), "dbName is null"); - java.util.Objects.requireNonNull(tableNameInfo.getCtl(), "catalogName is null"); - this.tableName = tableNameInfo.getTbl(); - this.dbName = tableNameInfo.getDb(); - this.ctlName = tableNameInfo.getCtl(); + this(java.util.Objects.requireNonNull(tableNameInfo, "tableNameInfo is null").getCtl(), + tableNameInfo.getDb(), tableNameInfo.getTbl()); + } + + /** Build an identity from canonical metadata names without SQL name normalization. */ + public BaseTableInfo(String catalogName, String databaseName, String tableName) { + this.ctlName = java.util.Objects.requireNonNull(catalogName, "catalogName is null"); + this.dbName = java.util.Objects.requireNonNull(databaseName, "dbName is null"); + this.tableName = java.util.Objects.requireNonNull(tableName, "tableName is null"); } // for replay MTMV, can not use `table.getDatabase();`,because database not added to catalog diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java index 21611d16cb3e7d..cc003146dc00b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java @@ -70,6 +70,9 @@ public class MTMVRelationManager implements MTMVHookService { // `tableMTMVs` will have 3 pair: table1 ==> mv1, mv1==>mv2, table1 ==> mv2 // `tableMTMVsOneLevelAndFromView` will have 2 pair: table1 ==> mv1, mv1==>mv2 // `viewMTMVs` will have 2 pair: v1 ==> mv1, v2 ==> mv1 + // Only dependency replacement and invalidation discovery use this lock. Release it before + // resolving tables or taking an MV lock: refresh completion can already hold that MV lock. + private final Object relationLock = new Object(); private final Map> tableMTMVs = Maps.newConcurrentMap(); private final Map> tableMTMVsOneLevelAndFromView = Maps.newConcurrentMap(); // view => mtmv @@ -85,24 +88,7 @@ public Set getMtmvsByBaseView(BaseTableInfo table) { /** Only discard local rewrite plans; replay must not refresh an MV or load external metadata. */ public void invalidateRewriteCaches(String catalog, String db, String table) { - Set affected = Sets.newHashSet(); - if (table != null) { - BaseTableInfo base = new BaseTableInfo(new TableNameInfo(catalog, db, table)); - affected.addAll(getMtmvsByBaseTable(base)); - affected.addAll(getMtmvsByBaseView(base)); - if (InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalog)) { - affected.add(base); - } - } else { - for (Map> relations : ImmutableList.of(tableMTMVs, viewMTMVs)) { - for (Map.Entry> entry : relations.entrySet()) { - BaseTableInfo base = entry.getKey(); - if (catalog.equals(base.getCtlName()) && (db == null || db.equals(base.getDbName()))) { - affected.addAll(entry.getValue()); - } - } - } - } + Set affected = collectAffectedMTMVs(catalog, db, table); List targets = Lists.newArrayList(); for (BaseTableInfo mv : affected) { Database database = Env.getCurrentInternalCatalog().getDbNullable(mv.getDbName()); @@ -122,6 +108,30 @@ public void invalidateRewriteCaches(String catalog, String db, String table) { } } + private Set collectAffectedMTMVs(String catalog, String db, String table) { + synchronized (relationLock) { + Set affected = Sets.newHashSet(); + if (table != null) { + BaseTableInfo base = new BaseTableInfo(catalog, db, table); + affected.addAll(getMtmvsByBaseTable(base)); + affected.addAll(getMtmvsByBaseView(base)); + if (InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalog)) { + affected.add(base); + } + } else { + for (Map> relations : ImmutableList.of(tableMTMVs, viewMTMVs)) { + for (Map.Entry> entry : relations.entrySet()) { + BaseTableInfo base = entry.getKey(); + if (catalog.equals(base.getCtlName()) && (db == null || db.equals(base.getDbName()))) { + affected.addAll(entry.getValue()); + } + } + } + } + return affected; + } + } + public Set getMtmvsByBaseTableOneLevelAndFromView(BaseTableInfo table) { return tableMTMVsOneLevelAndFromView.getOrDefault(table, ImmutableSet.of()); } @@ -206,8 +216,14 @@ private Set getOrCreateMTMVsOneLevelAndFromView(BaseTableInfo bas public void refreshMTMVCache(MTMVRelation relation, BaseTableInfo mtmvInfo) { LOG.info("refreshMTMVCache,relation: {}, mtmvInfo: {}", relation, mtmvInfo); - removeMTMV(mtmvInfo); - addMTMV(relation, mtmvInfo); + updateMTMVRelations(relation, mtmvInfo); + } + + private void updateMTMVRelations(MTMVRelation relation, BaseTableInfo mtmvInfo) { + synchronized (relationLock) { + removeMTMV(mtmvInfo); + addMTMV(relation, mtmvInfo); + } } private void addMTMV(MTMVRelation relation, BaseTableInfo mtmvInfo) { @@ -276,7 +292,7 @@ public void registerMTMV(MTMV mtmv, Long dbId) { */ @Override public void unregisterMTMV(MTMV mtmv) { - removeMTMV(new BaseTableInfo(mtmv)); + updateMTMVRelations(null, new BaseTableInfo(mtmv)); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteInvalidationRaceTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteInvalidationRaceTest.java new file mode 100644 index 00000000000000..0f8801155cf5d5 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteInvalidationRaceTest.java @@ -0,0 +1,250 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.info.TableNameInfo; + +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.util.AbstractSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +class MTMVRewriteInvalidationRaceTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void tableInvalidationPreservesCanonicalExternalNames(boolean globalLowerCase) { + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getName()).thenReturn("lake"); + Mockito.when(catalog.getLowerCaseTableNames()).thenReturn(2); + ExternalDatabase database = Mockito.mock(ExternalDatabase.class); + Mockito.doReturn(catalog).when(database).getCatalog(); + Mockito.when(database.getFullName()).thenReturn("source"); + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.doReturn(database).when(table).getDatabase(); + Mockito.when(table.getName()).thenReturn("Orders"); + MTMVRelationManager manager = new MTMVRelationManager(); + MTMV view = new MTMV(); + MTMVCache cached = Mockito.mock(MTMVCache.class); + Deencapsulation.setField(view, "cacheWithGuard", cached); + Deencapsulation.setField(view, "cacheWithoutGuard", cached); + try (MockedStatic env = localView(view)) { + env.when(Env::isStoredTableNamesLowerCase).thenReturn(globalLowerCase); + BaseTableInfo source = new BaseTableInfo(table); + Assertions.assertEquals("Orders", source.getTableName()); + Assertions.assertEquals(2, catalog.getLowerCaseTableNames()); + manager.refreshMTMVCache(relation(source, false), name("internal", "views", "dependent")); + + manager.invalidateRewriteCaches("lake", "source", "Orders"); + + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithGuard")); + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithoutGuard")); + } + } + + @ParameterizedTest + @CsvSource({"table,false", "database,false", "catalog,false", "table,true", "database,true", "catalog,true"}) + void invalidationCannotMissAMaterializedViewDuringDependencyReplacement(String scope, boolean baseView) + throws Exception { + MTMVRelationManager manager = new MTMVRelationManager(); + BaseTableInfo source = name("lake", "source", "t"); + BaseTableInfo viewName = name("internal", "views", "dependent"); + MTMVRelation relation = relation(source, baseView); + manager.refreshMTMVCache(relation, viewName); + Map> index = Deencapsulation.getField(manager, + baseView ? "viewMTMVs" : "tableMTMVs"); + PausingRemovalSet dependencies = new PausingRemovalSet(index.get(source)); + index.put(source, dependencies); + MTMV view = new MTMV(); + MTMVCache cached = Mockito.mock(MTMVCache.class); + AtomicReference updaterThread = new AtomicReference<>(); + AtomicReference invalidatorThread = new AtomicReference<>(); + CountDownLatch invalidationStarted = new CountDownLatch(1); + ExecutorService workers = Executors.newFixedThreadPool(2); + try { + Future update = workers.submit(() -> { + updaterThread.set(Thread.currentThread()); + view.writeMvLock(); + try { + // addTaskResult publishes its plan before replacing the dependency index. + Deencapsulation.setField(view, "cacheWithGuard", cached); + Deencapsulation.setField(view, "cacheWithoutGuard", cached); + manager.refreshMTMVCache(relation, viewName); + } finally { + view.writeMvUnlock(); + } + }); + Assertions.assertTrue(dependencies.removed.await(5, TimeUnit.SECONDS)); + Future invalidate = workers.submit(() -> { + invalidatorThread.set(Thread.currentThread()); + try (MockedStatic ignored = localView(view)) { + invalidationStarted.countDown(); + manager.invalidateRewriteCaches("lake", scope.equals("catalog") ? null : "source", + scope.equals("table") ? "t" : null); + } + }); + Assertions.assertTrue(invalidationStarted.await(5, TimeUnit.SECONDS)); + awaitDiscoveryOrIndexLock(invalidate, invalidatorThread.get(), updaterThread.get()); + dependencies.continueUpdate.countDown(); + update.get(5, TimeUnit.SECONDS); + invalidate.get(5, TimeUnit.SECONDS); + + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithGuard")); + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithoutGuard")); + Assertions.assertEquals(1L, (long) Deencapsulation.getField(view, "rewriteCacheGeneration")); + } finally { + dependencies.continueUpdate.countDown(); + workers.shutdownNow(); + Assertions.assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void cacheInvalidationDoesNotHoldTheDependencyIndexLock() throws Exception { + MTMVRelationManager manager = new MTMVRelationManager(); + BaseTableInfo source = name("lake", "source", "t"); + manager.refreshMTMVCache(relation(source, false), name("internal", "views", "dependent")); + MTMV view = Mockito.mock(MTMV.class); + CountDownLatch cacheInvalidation = new CountDownLatch(1); + CountDownLatch finishInvalidation = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + cacheInvalidation.countDown(); + Assertions.assertTrue(finishInvalidation.await(5, TimeUnit.SECONDS)); + return null; + }).when(view).invalidateRewriteCache(); + ExecutorService workers = Executors.newFixedThreadPool(2); + try { + Future invalidate = workers.submit(() -> { + try (MockedStatic ignored = localView(view)) { + manager.invalidateRewriteCaches("lake", "source", "t"); + } + }); + Assertions.assertTrue(cacheInvalidation.await(5, TimeUnit.SECONDS)); + // Refresh completion can already own an MV write lock. It must still update the index + // while another invalidation waits for that MV lock. + workers.submit(() -> manager.refreshMTMVCache(relation(source, false), + name("internal", "views", "another"))).get(5, TimeUnit.SECONDS); + finishInvalidation.countDown(); + invalidate.get(5, TimeUnit.SECONDS); + } finally { + finishInvalidation.countDown(); + workers.shutdownNow(); + Assertions.assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private static void awaitDiscoveryOrIndexLock(Future discovery, Thread reader, Thread writer) + throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (System.nanoTime() < deadline) { + if (discovery.isDone()) { + // The unfenced implementation finishes the empty lookup before registration resumes. + return; + } + ThreadInfo info = ManagementFactory.getThreadMXBean().getThreadInfo(reader.getId()); + if (info != null && info.getLockOwnerId() == writer.getId()) { + return; + } + TimeUnit.MILLISECONDS.sleep(1); + } + Assertions.fail("invalidation neither completed discovery nor waited for dependency replacement"); + } + + private static class PausingRemovalSet extends AbstractSet { + private final Set entries = ConcurrentHashMap.newKeySet(); + private final CountDownLatch removed = new CountDownLatch(1); + private final CountDownLatch continueUpdate = new CountDownLatch(1); + + PausingRemovalSet(Set entries) { + this.entries.addAll(entries); + } + + @Override + public boolean remove(Object entry) { + boolean changed = entries.remove(entry); + removed.countDown(); + try { + Assertions.assertTrue(continueUpdate.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + return changed; + } + + @Override + public boolean add(BaseTableInfo entry) { + return entries.add(entry); + } + + @Override + public Iterator iterator() { + return entries.iterator(); + } + + @Override + public int size() { + return entries.size(); + } + } + + private static BaseTableInfo name(String catalog, String db, String table) { + return new BaseTableInfo(new TableNameInfo(catalog, db, table)); + } + + private static MTMVRelation relation(BaseTableInfo source, boolean view) { + Set tables = view ? ImmutableSet.of() : ImmutableSet.of(source); + Set views = view ? ImmutableSet.of(source) : ImmutableSet.of(); + return new MTMVRelation(tables, tables, tables, views, views); + } + + private static MockedStatic localView(MTMV view) { + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + Mockito.when(catalog.getDbNullable("views")).thenReturn(database); + Mockito.when(database.getTableNullable("dependent")).thenReturn(view); + MockedStatic env = Mockito.mockStatic(Env.class); + env.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + return env; + } +} From 4dd904ad0a7e74f9ed416cd0cda93e31de6fa3f2 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 12 Nov 2026 18:52:00 +0800 Subject: [PATCH 3/4] [fix](fe) Separate database cache eviction from metadata changes ### What problem does this PR solve? Related PR: #63612 Problem Summary: With a one-entry external database object cache, constructing an MV plan that references two databases evicts a database object. The eviction was reported as a metadata change, invalidating the MV generation and restarting every successful build indefinitely. Separate automatic local-object retirement from explicit metadata invalidation while retaining typed engine cleanup and row-count fencing. Bound rewrite-cache construction to four attempts so sustained metadata changes fall back through the callers' existing optional-optimization exception handling. ### Release note Avoid repeated query planning caused by external database object cache eviction during materialized view rewriting. ### Check List (For Author) - Test: Unit Test - 192 focused FE tests passed with zero failures, errors or skips; FE build and integrated Checkstyle passed. The unfixed 4.1 reproduction had 6 failures among 11 tests. - Behavior changed: Yes - automatic database object eviction no longer reports a semantic metadata change, and unstable optional MV cache construction terminates after four attempts. - Does this need documentation: No --- .../java/org/apache/doris/catalog/MTMV.java | 6 +- .../doris/datasource/ExternalCatalog.java | 13 +- .../doris/datasource/ExternalDatabase.java | 24 +- .../datasource/ExternalMetaCacheMgr.java | 10 +- .../MTMVRewriteCacheInvalidationTest.java | 25 ++ .../doris/datasource/CatalogMgrTest.java | 2 +- ...xternalDatabaseEvictionQueryCacheTest.java | 262 ++++++++++++++++++ 7 files changed, 329 insertions(+), 13 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseEvictionQueryCacheTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index fe07e66bcab5a6..325838a2549d1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -69,6 +69,7 @@ public class MTMV extends OlapTable { private static final Logger LOG = LogManager.getLogger(MTMV.class); + private static final int MAX_REWRITE_CACHE_BUILD_ATTEMPTS = 4; private ReentrantReadWriteLock mvRwLock; @SerializedName("ri") @@ -489,7 +490,7 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws boolean sessionVarsMatch = SessionVarGuardRewriter.checkSessionVariablesMatch( currentSessionVars, this.sessionVariables); - while (true) { + for (int attempt = 0; attempt < MAX_REWRITE_CACHE_BUILD_ATTEMPTS; attempt++) { long cacheGeneration; readMvLock(); try { @@ -524,6 +525,9 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws writeMvUnlock(); } } + // MV rewriting and derived traits are optional; callers already skip them on analysis failure. + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "Failed to build rewrite cache for MV " + getName() + " because metadata kept changing"); } /** Invalidate both cache variants and reject any older build still in progress. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index c8f4c8d0f61739..b667ead3033d46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -1467,9 +1467,16 @@ boolean shouldInvalidateRoutedCacheOnDatabaseRemoval() { void handleDatabaseMetaCacheRemoval(Optional> value, RemovalCause cause) { - value.ifPresent(v -> v.resetMetaToUninitialized( - shouldInvalidateRoutedCacheOnDatabaseRemoval(), - !cause.wasEvicted() && shouldInvalidateRowCountOnDatabaseRemoval())); + value.ifPresent(database -> { + if (cause.wasEvicted()) { + // MetaCache objects are local representatives. Capacity/expiry retirement is not + // a schema or data change and must not invalidate a query that is loading its DBs. + database.resetMetaToUninitializedOnEviction(shouldInvalidateRoutedCacheOnDatabaseRemoval()); + } else { + database.resetMetaToUninitialized(shouldInvalidateRoutedCacheOnDatabaseRemoval(), + shouldInvalidateRowCountOnDatabaseRemoval()); + } + }); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 76c65f04abbe6a..4202e8c212638c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -128,6 +128,22 @@ public void resetMetaToUninitialized(boolean invalidateEngineCache) { } public void resetMetaToUninitialized(boolean invalidateEngineCache, boolean invalidateRowCountCache) { + resetMetaToUninitialized(invalidateRowCountCache, () -> { + if (invalidateEngineCache) { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(this); + } + }); + } + + void resetMetaToUninitializedOnEviction(boolean invalidateEngineCache) { + resetMetaToUninitialized(false, () -> { + if (invalidateEngineCache) { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbMetadataCache(this); + } + }); + } + + private void resetMetaToUninitialized(boolean invalidateRowCountCache, Runnable engineInvalidation) { if (LOG.isDebugEnabled()) { LOG.debug("resetToUninitialized db name {}, id {}, isInitializing: {}, initialized: {}", this.name, this.id, isInitializing, initialized, new Exception()); @@ -156,12 +172,8 @@ public void resetMetaToUninitialized(boolean invalidateEngineCache, boolean inva } } try { - if (invalidateEngineCache) { - // Route through the typed overload: connector-specific caches (for example Paimon's - // table loader) are keyed by the database object and are not fully covered by the - // name-based scan in invalidateDb(long, String). - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(this); - } + // Keep typed database identity for connector caches keyed by the retired object. + engineInvalidation.run(); } finally { if (invalidateRowCountCache) { // Independent of the routed invalidation: a connector cache failure (for example diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index a4d42930d47f25..15d28200ff866e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -609,8 +609,7 @@ public void invalidateDb(ExternalDatabase database) { long catalogId = database.getCatalog().getId(); CatalogIf owner = getCatalog(catalogId); try { - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "invalidateDb", () -> cache.invalidateDb(database))); + invalidateDbMetadataCache(database); } finally { if (owner != null) { Env.getCurrentEnv().onDatabaseChanged(owner.getName(), database.getFullName()); @@ -618,6 +617,13 @@ public void invalidateDb(ExternalDatabase database) { } } + /** Retire engine entries for a local DB object without reporting a metadata change. */ + void invalidateDbMetadataCache(ExternalDatabase database) { + long catalogId = database.getCatalog().getId(); + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "invalidateDb", () -> cache.invalidateDb(database))); + } + public void invalidateTable(long catalogId, String dbName, String tableName) { invalidateTableCaches(catalogId, dbName, tableName, () -> { invalidateLanceTableAccess(catalogId); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/MTMVRewriteCacheInvalidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/MTMVRewriteCacheInvalidationTest.java index c1b1bcc824a9ba..2ccd69b9319eaa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/MTMVRewriteCacheInvalidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/MTMVRewriteCacheInvalidationTest.java @@ -38,6 +38,31 @@ import java.util.concurrent.atomic.AtomicInteger; class MTMVRewriteCacheInvalidationTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void repeatedMetadataChangesBoundCacheConstruction(boolean sameSessionVariables) { + AtomicInteger builds = new AtomicInteger(); + MTMVCache stale = Mockito.mock(MTMVCache.class); + MTMV view = new MTMV() { + @Override + protected MTMVCache createRewriteCache(ConnectContext context, boolean needLock, boolean addGuard) { + Assertions.assertTrue(builds.incrementAndGet() <= 8, "cache construction retried indefinitely"); + invalidateRewriteCache(); + return stale; + } + }; + if (!sameSessionVariables) { + Deencapsulation.setField(view, "sessionVariables", ImmutableMap.of("test", "different")); + } + ConnectContext connection = Mockito.mock(ConnectContext.class); + Mockito.when(connection.getSessionVariable()).thenReturn(new SessionVariable()); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> view.getOrGenerateCache(connection)); + Assertions.assertTrue(builds.get() <= 4); + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithGuard")); + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithoutGuard")); + } + @ParameterizedTest @ValueSource(booleans = {false, true}) void invalidationRejectsAnOlderBuildForBothCacheVariants(boolean sameSessionVariables) throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 0fa144a6ef26c9..f631396ea18b47 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -75,7 +75,7 @@ void testDatabaseCapacityEvictionDoesNotFenceAllRowCounts() { ExternalDatabase db = Mockito.mock(ExternalDatabase.class); catalog.handleDatabaseMetaCacheRemoval(Optional.of(db), RemovalCause.SIZE); - Mockito.verify(db).resetMetaToUninitialized(true, false); + Mockito.verify(db).resetMetaToUninitializedOnEviction(true); catalog.handleDatabaseMetaCacheRemoval(Optional.of(db), RemovalCause.EXPLICIT); Mockito.verify(db).resetMetaToUninitialized(true, true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseEvictionQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseEvictionQueryCacheTest.java new file mode 100644 index 00000000000000..ad0e7934d4055b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseEvictionQueryCacheTest.java @@ -0,0 +1,262 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.common.Pair; +import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; +import org.apache.doris.common.cache.NereidsSqlCacheManager; +import org.apache.doris.common.cache.SqlCacheMetadataVersions; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.metacache.ExternalMetaCache; +import org.apache.doris.datasource.metacache.MetaCache; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVCache; +import org.apache.doris.mtmv.MTMVRefreshSnapshot; +import org.apache.doris.mtmv.MTMVRelation; +import org.apache.doris.mtmv.MTMVRelationManager; +import org.apache.doris.mtmv.MTMVService; +import org.apache.doris.mtmv.MTMVStatus; +import org.apache.doris.nereids.SqlCacheContext.FullTableName; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.StatementContext.TableFrom; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; + +class ExternalDatabaseEvictionQueryCacheTest { + @ParameterizedTest + @ValueSource(ints = {1, 2}) + void crossDatabaseCacheBuildSurvivesObjectCapacityPressure(int capacity) { + Fixture fixture = new Fixture(capacity); + AtomicInteger builds = new AtomicInteger(); + MTMVCache generated = Mockito.mock(MTMVCache.class); + MTMV view = new MTMV() { + @Override + protected MTMVCache createRewriteCache(ConnectContext connection, boolean needLock, boolean addGuard) { + // Stop the old unbounded loop deterministically rather than leaking a planner thread. + Assertions.assertTrue(builds.incrementAndGet() <= 8, "MV cache build did not converge"); + try (StatementContext statement = new StatementContext(connection, null)) { + for (String db : ImmutableList.of("one", "two")) { + Assertions.assertNotNull(statement.getAndCacheTable( + ImmutableList.of("lake", db, "t"), TableFrom.QUERY, Optional.empty())); + } + fixture.catalog.objects.getMetaObjCache().cleanUp(); + } + return generated; + } + }; + fixture.setView(view); + try (MockedStatic ignored = fixture.currentEnv()) { + SqlCacheMetadataVersions.Snapshot snapshot = fixture.snapshot(); + Assertions.assertSame(generated, view.getOrGenerateCache(fixture.connection)); + Assertions.assertSame(generated, view.getOrGenerateCache(fixture.connection)); + Assertions.assertEquals(1, builds.get()); + Assertions.assertTrue(snapshot.isCurrent(), "object eviction is not a metadata mutation"); + if (capacity == 1) { + Assertions.assertTrue(fixture.catalog.evictions.get() > 0, "the real DB cache must evict"); + Mockito.verify(fixture.engine, Mockito.atLeastOnce()).invalidateDb(Mockito.any(ExternalDatabase.class)); + } else { + Assertions.assertEquals(0, fixture.catalog.evictions.get()); + } + } + } + + @ParameterizedTest + @EnumSource(value = RemovalCause.class, names = {"SIZE", "EXPIRED", "COLLECTED"}) + void automaticRemovalRetiresMetadataWithoutInvalidatingQueryDependencies(RemovalCause cause) { + Fixture fixture = new Fixture(2); + MTMV view = new MTMV(); + fixture.setView(view); + MTMVCache cached = Mockito.mock(MTMVCache.class); + Deencapsulation.setField(view, "cacheWithGuard", cached); + Deencapsulation.setField(view, "cacheWithoutGuard", cached); + try (MockedStatic ignored = fixture.currentEnv()) { + ExternalDatabase db = fixture.catalog.getDbNullable("one"); + SqlCacheMetadataVersions.Snapshot snapshot = fixture.snapshot(); + + fixture.catalog.handleDatabaseMetaCacheRemoval(Optional.of(db), cause); + + Mockito.verify(fixture.engine).invalidateDb(db); + Assertions.assertTrue(snapshot.isCurrent()); + Assertions.assertSame(cached, Deencapsulation.getField(view, "cacheWithGuard")); + Assertions.assertSame(cached, Deencapsulation.getField(view, "cacheWithoutGuard")); + } + } + + @ParameterizedTest + @EnumSource(value = RemovalCause.class, names = {"EXPLICIT", "REPLACED"}) + void explicitRemovalStillInvalidatesQueryDependencies(RemovalCause cause) { + Fixture fixture = new Fixture(2); + MTMV view = new MTMV(); + fixture.setView(view); + MTMVCache cached = Mockito.mock(MTMVCache.class); + Deencapsulation.setField(view, "cacheWithGuard", cached); + Deencapsulation.setField(view, "cacheWithoutGuard", cached); + try (MockedStatic ignored = fixture.currentEnv()) { + ExternalDatabase db = fixture.catalog.getDbNullable("one"); + SqlCacheMetadataVersions.Snapshot snapshot = fixture.snapshot(); + + fixture.catalog.handleDatabaseMetaCacheRemoval(Optional.of(db), cause); + + Mockito.verify(fixture.engine).invalidateDb(db); + Assertions.assertFalse(snapshot.isCurrent()); + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithGuard")); + Assertions.assertNull(Deencapsulation.getField(view, "cacheWithoutGuard")); + } + } + + private static class Fixture { + private final Env env = Mockito.mock(Env.class, Mockito.CALLS_REAL_METHODS); + private final EvictingCatalog catalog; + private final NereidsSqlCacheManager sql = new NereidsSqlCacheManager(); + private final MTMVRelationManager relations = new MTMVRelationManager(); + private final InternalCatalog internal = Mockito.mock(InternalCatalog.class); + private final Database views = Mockito.mock(Database.class); + private final ExternalMetaCache engine = Mockito.mock(ExternalMetaCache.class); + private final ConnectContext connection = Mockito.mock(ConnectContext.class); + + Fixture(int capacity) { + catalog = new EvictingCatalog(capacity); + CatalogMgr catalogs = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogs).getCatalog("lake"); + Mockito.doReturn(catalog).when(catalogs).getCatalog(catalog.getId()); + Mockito.doReturn(catalogs).when(env).getCatalogMgr(); + Mockito.doReturn(sql).when(env).getSqlCacheManager(); + Mockito.doReturn(Mockito.mock(NereidsSortedPartitionsCacheManager.class)) + .when(env).getSortedPartitionsCacheManager(); + MTMVService service = Mockito.mock(MTMVService.class); + Mockito.when(service.getRelationManager()).thenReturn(relations); + Mockito.doReturn(service).when(env).getMtmvService(); + Mockito.when(internal.getDbNullable("views")).thenReturn(views); + Mockito.when(connection.getEnv()).thenReturn(env); + Mockito.when(connection.getCurrentUserIdentity()).thenReturn(UserIdentity.ROOT); + Mockito.when(connection.getSessionVariable()).thenReturn(new SessionVariable()); + Mockito.when(engine.engine()).thenReturn("default"); + Mockito.when(engine.isCatalogInitialized(catalog.getId())).thenReturn(true); + ExternalMetaCacheMgr caches = new ExternalMetaCacheMgr(true); + caches.replaceEngineCachesForTest(ImmutableList.of(engine)); + Mockito.doReturn(caches).when(env).getExtMetaCacheMgr(); + } + + void setView(MTMV view) { + Deencapsulation.setField(view, "status", new MTMVStatus()); + Deencapsulation.setField(view, "refreshSnapshot", new MTMVRefreshSnapshot()); + Mockito.when(views.getTableNullable("dependent")).thenReturn(view); + ImmutableSet sources = ImmutableSet.of( + new BaseTableInfo("lake", "one", "t"), new BaseTableInfo("lake", "two", "t")); + relations.refreshMTMVCache(new MTMVRelation(sources, sources, sources, + ImmutableSet.of(), ImmutableSet.of()), new BaseTableInfo("internal", "views", "dependent")); + } + + SqlCacheMetadataVersions.Snapshot snapshot() { + SqlCacheMetadataVersions.Snapshot snapshot = sql.newMetadataSnapshot(); + for (String db : ImmutableList.of("one", "two")) { + snapshot.addTable(Util.genIdByName("lake", db, "t"), new FullTableName("lake", db, "t")); + } + return snapshot; + } + + MockedStatic currentEnv() { + MockedStatic current = Mockito.mockStatic(Env.class); + current.when(Env::getCurrentEnv).thenReturn(env); + current.when(Env::getCurrentInternalCatalog).thenReturn(internal); + return current; + } + } + + private static class EvictingCatalog extends ExternalCatalog { + private final MetaCache> objects; + private final AtomicInteger evictions = new AtomicInteger(); + + EvictingCatalog(int capacity) { + super(1L, "lake", InitCatalogLog.Type.TEST, ""); + catalogProperty = new CatalogProperty(null, Collections.emptyMap()); + objects = new MetaCache<>("databases", MoreExecutors.newDirectExecutorService(), + OptionalLong.empty(), OptionalLong.empty(), capacity, + ignored -> ImmutableList.of(Pair.of("one", "one"), Pair.of("two", "two")), + db -> Optional.of(new DummyDatabase(this, db)), + (key, value, cause) -> { + if (cause.wasEvicted()) { + evictions.incrementAndGet(); + } + handleDatabaseMetaCacheRemoval(value, cause); + }); + metaCache = objects; + initialized = true; + } + + @Override + protected List listTableNamesFromRemote(SessionContext context, String db) { + return ImmutableList.of("t"); + } + + @Override + public boolean tableExist(SessionContext context, String db, String table) { + return true; + } + + @Override + protected void initLocalObjectsImpl() { + } + } + + private static class DummyDatabase extends ExternalDatabase { + private final ExternalTable table; + + DummyDatabase(ExternalCatalog catalog, String name) { + super(catalog, Util.genIdByName(catalog.getName(), name), name, name, InitDatabaseLog.Type.TEST); + table = Mockito.mock(ExternalTable.class); + Mockito.when(table.getName()).thenReturn("t"); + Mockito.when(table.getId()).thenReturn(Util.genIdByName(catalog.getName(), name, "t")); + Mockito.doReturn(this).when(table).getDatabase(); + Mockito.doReturn(catalog).when(table).getCatalog(); + } + + @Override + public ExternalTable getTableNullable(String name) { + return table; + } + + @Override + protected ExternalTable buildTableInternal(String remoteName, String localName, long id, + ExternalCatalog catalog, ExternalDatabase database) { + return table; + } + } +} From 742a21e1d2d365dcf344f57e56ac1704c34e63ba Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 13 Nov 2026 16:56:39 +0800 Subject: [PATCH 4/4] [test](regression) Rewarm SQL caches after rollup publication ### What problem does this PR solve? Related PR: #63612 Problem Summary: The synchronous-MV cache regression warmed base-table queries before publishing rollups and expected their old SQL cache entries to survive. Rollup publication now notifies table metadata changes and correctly invalidates those entries. Share one check across the rename, insert and drop scenarios to verify cache misses after publication, then rerun each query and verify cache hits. Preserve the later rename, insert and drop assertions. ### Release note None ### Check List (For Author) - Test: 40 focused FE unit tests passed; regression framework and modified Groovy suite compiled; FE build with integrated Checkstyle passed. End-to-end regression was not run locally because this worktree has no BE binary. - Behavior changed: No production behavior change; update regression expectations for metadata invalidation. - Does this need documentation: No --- .../nereids_p0/cache/mv_with_sql_cache.groovy | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/regression-test/suites/nereids_p0/cache/mv_with_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/mv_with_sql_cache.groovy index 6b9fe0d656deac..30545374f45e77 100644 --- a/regression-test/suites/nereids_p0/cache/mv_with_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/mv_with_sql_cache.groovy @@ -60,6 +60,15 @@ suite("mv_with_sql_cache") { } } + // Publishing a rollup changes the table metadata and invalidates the old cached plans. + def assertCacheInvalidatedAndRebuilt = { List queries -> + queries.each { assertNoCache(it) } + queries.each { query -> + sql(query) + assertHasCache(query) + } + } + def retryTestSqlCache = { int executeTimes, long intervalMillis, Closure closure -> Throwable throwable = null for (int i = 1; i <= executeTimes; ++i) { @@ -118,9 +127,7 @@ suite("mv_with_sql_cache") { create_sync_mv(dbName, tb_name1, mv_name2, mv_sql2) sleep(10 * 1000) - assertHasCache "select * from ${tb_name1}" - assertHasCache mv_sql1 - assertHasCache mv_sql2 + assertCacheInvalidatedAndRebuilt(["select * from ${tb_name1}", mv_sql1, mv_sql2]) assertNoCache mv_str sql mv_str @@ -181,9 +188,7 @@ suite("mv_with_sql_cache") { create_sync_mv(dbName, tb_name1, mv_name2, mv_sql2) sleep(10 * 1000) - assertHasCache "select * from ${tb_name1}" - assertHasCache mv_sql1 - assertHasCache mv_sql2 + assertCacheInvalidatedAndRebuilt(["select * from ${tb_name1}", mv_sql1, mv_sql2]) assertNoCache mv_str sql mv_str @@ -244,9 +249,7 @@ suite("mv_with_sql_cache") { create_sync_mv(dbName, tb_name1, mv_name2, mv_sql2) sleep(10 * 1000) - assertHasCache "select * from ${tb_name1}" - assertHasCache mv_sql1 - assertHasCache mv_sql2 + assertCacheInvalidatedAndRebuilt(["select * from ${tb_name1}", mv_sql1, mv_sql2]) assertNoCache mv_str sql mv_str