From 4613184eb5a8f87aab974c1324ff0ff6f2cad773 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Sun, 20 Sep 2026 17:35:48 +0800 Subject: [PATCH 1/2] [fix](mtmv) Preserve origin statement for internal refresh ### What problem does this PR solve? Problem Summary: Internally constructed MTMV refresh commands could lose their origin statement. Complete refreshes created a StatementContext without the MV query, while StmtExecutor copied a null LogicalPlanAdapter origin back into contexts that already had one. Initialize complete-refresh contexts with the MV query and propagate the same OriginStatement to the LogicalPlanAdapter before executor construction. ### Release note Fix internal materialized view refresh planning to retain its originating SQL statement. ### Check List (For Author) - Test: Unit Test - Behavior changed: No - Does this need documentation: No --- .../org/apache/doris/job/extensions/mtmv/MTMVTask.java | 8 +++++--- .../main/java/org/apache/doris/mtmv/MTMVPlanUtil.java | 7 ++++++- .../test/java/org/apache/doris/mtmv/MTMVTaskTest.java | 5 +++++ .../plans/commands/UpdateMvByPartitionCommandTest.java | 9 ++++++++- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 1eab83fe576d3e..b398d5ab892915 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -78,6 +78,7 @@ import org.apache.doris.nereids.trees.plans.commands.UpdateMvByPartitionCommand; import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo.RefreshMode; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.rpc.RpcException; import org.apache.doris.system.SystemInfoService; @@ -962,10 +963,11 @@ private IvmPlanSignature refreshPartitions(Set refreshPartitionNames, Map tableWithPartKey, Optional rewriteContext, RefreshMode refreshMode) throws Exception { - // Create MTMV context first so that new StatementContext() captures the - // correct thread-local ConnectContext (with MTMV disabled rules, etc.). + // Create the MTMV context before parsing the MV definition SQL so SET_VAR hints + // resolve against the internal session (with MTMV disabled rules, etc.). ConnectContext mtmvCtx = MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK); - StatementContext statementContext = new StatementContext(); + StatementContext statementContext = new StatementContext( + mtmvCtx, new OriginStatement(mtmv.getQuerySql(), 0)); // Install the StatementContext on the ConnectContext before parsing // the MV definition SQL. UpdateMvByPartitionCommand.from() calls // NereidsParser.parseSingle() which, for SQL containing SET_VAR hints, diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 45c528449a86a2..425018f0a31009 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -172,6 +172,8 @@ public static void executeCommand(MTMV mtmv, Command command, * executing {@link StmtExecutor} through {@code executorConsumer} before the command * runs and clearing it (with {@code null}) after the command finishes, so task * cancellation can interrupt the running statement. + * + *

The supplied statement context must contain the originating SQL statement. */ public static void executeCommand(ConnectContext ctx, Command command, StatementContext stmtCtx, @Nullable String auditStmt, @@ -180,7 +182,10 @@ public static void executeCommand(ConnectContext ctx, Command command, ctx.getState().setNereids(true); ctx.getSessionVariable().setEnableMaterializedViewRewrite(false); ctx.getSessionVariable().setEnableDmlMaterializedViewRewrite(false); - StmtExecutor executor = new StmtExecutor(ctx, new LogicalPlanAdapter(command, stmtCtx)); + LogicalPlanAdapter adapter = new LogicalPlanAdapter(command, stmtCtx); + adapter.setOrigStmt(Preconditions.checkNotNull(stmtCtx.getOriginStatement(), + "MTMV command origin statement must not be null")); + StmtExecutor executor = new StmtExecutor(ctx, adapter); ctx.setExecutor(executor); ctx.setQueryId(AbstractTask.generateQueryId()); if (executorConsumer != null) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java index 0f4d44634caeeb..8446069e9211de 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java @@ -477,6 +477,7 @@ public void testExecCarriesExcludedTriggerTablesIntoStatementContext() throws Ex Mockito.when(mtmv.getExcludedTriggerTables()).thenReturn(excludedTriggerTables); Mockito.when(mtmv.isIvm()).thenReturn(true); Mockito.when(mtmv.getName()).thenReturn("test_mv"); + Mockito.when(mtmv.getQuerySql()).thenReturn("select k1 from test_db.base_table"); Mockito.when(mtmv.getDatabase()).thenReturn(null); Mockito.when(mtmvPartitionInfo.getPartitionType()).thenReturn(MTMVPartitionType.FOLLOW_BASE_TABLE); @@ -502,6 +503,8 @@ public void testExecCarriesExcludedTriggerTablesIntoStatementContext() throws Ex public UpdateMvByPartitionCommand answer(InvocationOnMock invocation) { StatementContext statementContext = invocation.getArgument(3); Assertions.assertEquals(excludedTriggerTables, statementContext.getExcludedTriggerTables()); + Assertions.assertEquals("select k1 from test_db.base_table", + statementContext.getOriginStatement().originStmt); return command; } }); @@ -512,6 +515,8 @@ public UpdateMvByPartitionCommand answer(InvocationOnMock invocation) { public Void answer(InvocationOnMock invocation) { StatementContext statementContext = invocation.getArgument(2); Assertions.assertEquals(excludedTriggerTables, statementContext.getExcludedTriggerTables()); + Assertions.assertEquals("select k1 from test_db.base_table", + statementContext.getOriginStatement().originStmt); return null; } }); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java index 0eb933b45a4218..4228865cfa4902 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java @@ -46,6 +46,7 @@ import org.apache.doris.planner.ExchangeNode; import org.apache.doris.planner.OlapTableSink; import org.apache.doris.planner.PlanFragment; +import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.thrift.TPartitionType; import org.apache.doris.utframe.TestWithFeService; @@ -241,6 +242,7 @@ void testPlannerKeepsIvmRowIdAsBigIntInSinkTupleAndOutputExprs() throws Exceptio void testRunRefreshCommandExecutesIncrementalMtmv() throws Exception { MTMV mtmv = getMtmv("ivm_mv"); StatementContext statementContext = createStatementCtx("refresh materialized view test.ivm_mv"); + OriginStatement originStatement = statementContext.getOriginStatement(); statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.full(mtmv))); UpdateMvByPartitionCommand command = newRefreshCommand(mtmv); AtomicReference executorRef = new AtomicReference<>(); @@ -260,12 +262,15 @@ void testRunRefreshCommandExecutesIncrementalMtmv() throws Exception { executor.getContext().getStatementContext().getIvmRewriteContext().orElseThrow().getMode()); Assertions.assertSame(executor.getContext(), statementContext.getConnectContext()); Assertions.assertSame(statementContext, executor.getContext().getStatementContext()); + Assertions.assertSame(originStatement, statementContext.getOriginStatement()); + Assertions.assertSame(originStatement, executor.getParsedStmt().getOrigStmt()); } @Test void testExecuteCommandRebindsTaskStatementContextToExecutionContext() throws Exception { MTMV mtmv = getMtmv("ivm_mv"); - StatementContext statementContext = new StatementContext(); + StatementContext statementContext = createStatementCtx("refresh materialized view test.ivm_mv"); + OriginStatement originStatement = statementContext.getOriginStatement(); statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.full(mtmv))); UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand.from( mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext); @@ -281,6 +286,8 @@ void testExecuteCommandRebindsTaskStatementContextToExecutionContext() throws Ex Assertions.assertSame(executor.getContext(), statementContext.getConnectContext()); Assertions.assertSame(statementContext, executor.getContext().getStatementContext()); + Assertions.assertSame(originStatement, statementContext.getOriginStatement()); + Assertions.assertSame(originStatement, executor.getParsedStmt().getOrigStmt()); } @Test From 7cb38cef96b1ed820e7a12859518a692e8f72941 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Wed, 23 Sep 2026 17:04:16 +0800 Subject: [PATCH 2/2] [fix](mtmv) Install incremental refresh context before parsing hints ### What problem does this PR solve? Issue Number: None Related PR: #68269 Problem Summary: Incremental refresh and dry run parsed the MV query before installing their new StatementContext on the internal ConnectContext. Parsing a SET_VAR hint reads that context and could fail with a null dereference. Install the origin-bearing context before parsing in both paths, and cover the hinted MV query with FE and regression tests. ### Release note Fix incremental materialized view refresh and dry run for MV queries containing SET_VAR hints. ### Check List (For Author) - Test: Unit Test - Focused FE unit tests and Checkstyle passed. - Hinted dry-run regression case added; not run locally. - Behavior changed: Yes. Hinted incremental refreshes now parse successfully. - Does this need documentation: No --- .../doris/mtmv/ivm/IvmIncrRefreshManager.java | 2 ++ .../plans/commands/RefreshMTMVCommand.java | 3 +++ .../mtmv/ivm/IvmIncrRefreshManagerTest.java | 27 +++++++++++++++++++ .../commands/RefreshMTMVCommandTest.java | 18 +++++++++++++ .../ivm/test_ivm_refresh_dry_run.groovy | 2 +- 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmIncrRefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmIncrRefreshManager.java index 4559fd549370f3..cfbdff60d18a27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmIncrRefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmIncrRefreshManager.java @@ -97,6 +97,8 @@ void executeInternalRefresh(IvmIncrRefreshContext context) throws Exception { MTMV mtmv = context.getMtmv(); StatementContext statementContext = new StatementContext( context.getConnectContext(), new OriginStatement(mtmv.getQuerySql(), 0)); + // SET_VAR hints are applied while parsing the MV query, before executeCommand runs. + context.getConnectContext().setStatementContext(statementContext); // The delta may only read the base partitions the MV's partition definition keeps. A base // partition outside that set, expired by partition_sync_limit, would otherwise still be // read through the delta and the join-opposite snapshot, and its rows would have no MV diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java index 55f0e42e73538e..16507e80bd3cf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java @@ -152,6 +152,9 @@ StatementContext createDryRunStatementContext(MTMV mtmv, ConnectContext internal stmtCtx.setIvmRewriteContext(Optional.of(IvmRewriteContext.incrementalDryRun(mtmv, dryRunLimit))); // Excluded trigger tables must not be validated for binlog / key-type support. stmtCtx.setExcludedTriggerTables(mtmv.getExcludedTriggerTables()); + // The MV query is parsed before the internal executor is created. SET_VAR hints + // need this context already installed on the internal session during parsing. + internalCtx.setStatementContext(stmtCtx); return stmtCtx; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmIncrRefreshManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmIncrRefreshManagerTest.java index 3a4a4519e12403..59a57d660a0708 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmIncrRefreshManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmIncrRefreshManagerTest.java @@ -90,6 +90,33 @@ public void testExecuteInternalRefreshPassesExcludedTriggerTables() throws Excep Assertions.assertEquals(excluded, captured.get().getExcludedTriggerTables()); } + @Test + public void testIncrementalRefreshParsesSetVarWithItsStatementContext() throws Exception { + MTMV mtmv = mockMtmv(); + Mockito.when(mtmv.getQuerySql()).thenReturn("SELECT /*+ SET_VAR(query_timeout=10) */ 1 AS k1"); + Mockito.when(mtmv.getInsertedColumnNames()).thenReturn(List.of("k1")); + ConnectContext connectContext = new ConnectContext(); + IvmIncrRefreshContext context = new IvmIncrRefreshContext(mtmv, connectContext, "audit", + queryId -> { }, null); + connectContext.setThreadLocalInfo(); + try (MockedStatic mockedUtil = Mockito.mockStatic(MTMVPlanUtil.class)) { + mockedUtil.when(() -> MTMVPlanUtil.executeCommand( + Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())) + .thenAnswer(inv -> { + StatementContext stmtCtx = inv.getArgument(2); + Assertions.assertSame(stmtCtx, connectContext.getStatementContext()); + Assertions.assertEquals(mtmv.getQuerySql(), stmtCtx.getOriginStatement().originStmt); + Assertions.assertEquals(10, connectContext.getSessionVariable().getQueryTimeoutS()); + return null; + }); + new IvmIncrRefreshManager().executeInternalRefresh(context); + mockedUtil.verify(() -> MTMVPlanUtil.executeCommand( + Mockito.eq(connectContext), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())); + } finally { + ConnectContext.remove(); + } + } + @Test public void testManagerReturnsSuccessForEmptyBundles() throws Exception { MTMV mtmv = mockMtmv(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommandTest.java index 689d7fb0dadb24..81becc8b5a9c66 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommandTest.java @@ -172,6 +172,24 @@ public void testDryRunStatementContextCarriesExcludedTriggerTables() throws Exce Assertions.assertEquals(excluded, stmtCtx.getExcludedTriggerTables()); } + @Test + public void testIncrementalDryRunParsesSetVarWithItsStatementContext() throws Exception { + RefreshMTMVInfo info = extractRefreshInfo("REFRESH MATERIALIZED VIEW db1.mv1 INCREMENTAL"); + TestRefreshMTMVCommand command = new TestRefreshMTMVCommand(info, true); + MTMV mtmv = Mockito.mock(MTMV.class); + Mockito.when(mtmv.getQuerySql()).thenReturn("SELECT /*+ SET_VAR(query_timeout=10) */ 1 AS k1"); + ConnectContext internalCtx = new ConnectContext(); + internalCtx.setThreadLocalInfo(); + try { + StatementContext stmtCtx = command.createDryRunStatementContext(mtmv, internalCtx); + new IvmIncrRefreshManager().buildQueryPlan(mtmv); + Assertions.assertSame(stmtCtx, internalCtx.getStatementContext()); + Assertions.assertEquals(10, internalCtx.getSessionVariable().getQueryTimeoutS()); + } finally { + ConnectContext.remove(); + } + } + @Test public void testIncrementalExplainCarriesExcludedTriggerTables() throws Exception { RefreshMTMVInfo info = extractRefreshInfo("REFRESH MATERIALIZED VIEW db1.mv1 INCREMENTAL"); diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_refresh_dry_run.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_refresh_dry_run.groovy index f2583e14e8d333..073d400a1b510b 100644 --- a/regression-test/suites/mtmv_p0/ivm/test_ivm_refresh_dry_run.groovy +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_refresh_dry_run.groovy @@ -48,7 +48,7 @@ suite("test_ivm_refresh_dry_run") { BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') - AS SELECT k1, COUNT(*) AS cnt, SUM(v1) AS sum_v1 + AS SELECT /*+ SET_VAR(query_timeout=180) */ k1, COUNT(*) AS cnt, SUM(v1) AS sum_v1 FROM test_ivm_refresh_dry_run_base GROUP BY k1 """