Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
import org.apache.doris.nereids.trees.plans.commands.UpdateMvByPartitionCommand;
import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo.RefreshMode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
import org.apache.doris.qe.QeProcessorImpl;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.rpc.RpcException;
Expand Down Expand Up @@ -969,10 +970,11 @@ private IvmPlanSignature refreshPartitions(Set<String> refreshPartitionNames,
Map<TableIf, String> tableWithPartKey,
Optional<IvmRewriteContext> rewriteContext, RefreshMode refreshMode)
throws Exception {
// Create MTMV context first so that new StatementContext() captures the
// correct thread-local ConnectContext (with MTMV disabled rules, etc.).
// Create the MTMV context before parsing the MV definition SQL so SET_VAR hints
// resolve against the internal session (with MTMV disabled rules, etc.).
ConnectContext mtmvCtx = MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK);
StatementContext statementContext = new StatementContext();
StatementContext statementContext = new StatementContext(
mtmvCtx, new OriginStatement(mtmv.getQuerySql(), 0));
Comment thread
morrySnow marked this conversation as resolved.
Resolved
// Install the StatementContext on the ConnectContext before parsing
// the MV definition SQL. UpdateMvByPartitionCommand.from() calls
// NereidsParser.parseSingle() which, for SQL containing SET_VAR hints,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,8 @@ public static void executeCommand(MTMV mtmv, Command command,
* executing {@link StmtExecutor} through {@code executorConsumer} before the command
* runs and clearing it (with {@code null}) after the command finishes, so task
* cancellation can interrupt the running statement.
*
* <p>The supplied statement context must contain the originating SQL statement.
*/
public static void executeCommand(ConnectContext ctx, Command command,
StatementContext stmtCtx, @Nullable String auditStmt,
Expand All @@ -180,7 +182,10 @@ public static void executeCommand(ConnectContext ctx, Command command,
ctx.getState().setNereids(true);
ctx.getSessionVariable().setEnableMaterializedViewRewrite(false);
ctx.getSessionVariable().setEnableDmlMaterializedViewRewrite(false);
StmtExecutor executor = new StmtExecutor(ctx, new LogicalPlanAdapter(command, stmtCtx));
LogicalPlanAdapter adapter = new LogicalPlanAdapter(command, stmtCtx);
adapter.setOrigStmt(Preconditions.checkNotNull(stmtCtx.getOriginStatement(),
"MTMV command origin statement must not be null"));
StmtExecutor executor = new StmtExecutor(ctx, adapter);
ctx.setExecutor(executor);
ctx.setQueryId(AbstractTask.generateQueryId());
if (executorConsumer != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ void executeInternalRefresh(IvmIncrRefreshContext context) throws Exception {
MTMV mtmv = context.getMtmv();
StatementContext statementContext = new StatementContext(
context.getConnectContext(), new OriginStatement(mtmv.getQuerySql(), 0));
// SET_VAR hints are applied while parsing the MV query, before executeCommand runs.
context.getConnectContext().setStatementContext(statementContext);
Comment thread
morrySnow marked this conversation as resolved.
Resolved
// The delta may only read the base partitions the MV's partition definition keeps. A base
// partition outside that set, expired by partition_sync_limit, would otherwise still be
// read through the delta and the join-opposite snapshot, and its rows would have no MV
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,9 @@ StatementContext createDryRunStatementContext(MTMV mtmv, ConnectContext internal
stmtCtx.setIvmRewriteContext(Optional.of(IvmRewriteContext.incrementalDryRun(mtmv, dryRunLimit)));
// Excluded trigger tables must not be validated for binlog / key-type support.
stmtCtx.setExcludedTriggerTables(mtmv.getExcludedTriggerTables());
// The MV query is parsed before the internal executor is created. SET_VAR hints
// need this context already installed on the internal session during parsing.
internalCtx.setStatementContext(stmtCtx);
Comment thread
morrySnow marked this conversation as resolved.
Resolved
return stmtCtx;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,7 @@ public void testExecCarriesExcludedTriggerTablesIntoStatementContext() throws Ex
Mockito.when(mtmv.getExcludedTriggerTables()).thenReturn(excludedTriggerTables);
Mockito.when(mtmv.isIvm()).thenReturn(true);
Mockito.when(mtmv.getName()).thenReturn("test_mv");
Mockito.when(mtmv.getQuerySql()).thenReturn("select k1 from test_db.base_table");
Mockito.when(mtmv.getDatabase()).thenReturn(null);
Mockito.when(mtmvPartitionInfo.getPartitionType()).thenReturn(MTMVPartitionType.FOLLOW_BASE_TABLE);

Expand All @@ -502,6 +503,8 @@ public void testExecCarriesExcludedTriggerTablesIntoStatementContext() throws Ex
public UpdateMvByPartitionCommand answer(InvocationOnMock invocation) {
StatementContext statementContext = invocation.getArgument(3);
Assertions.assertEquals(excludedTriggerTables, statementContext.getExcludedTriggerTables());
Assertions.assertEquals("select k1 from test_db.base_table",
statementContext.getOriginStatement().originStmt);
return command;
}
});
Expand All @@ -512,6 +515,8 @@ public UpdateMvByPartitionCommand answer(InvocationOnMock invocation) {
public Void answer(InvocationOnMock invocation) {
StatementContext statementContext = invocation.getArgument(2);
Assertions.assertEquals(excludedTriggerTables, statementContext.getExcludedTriggerTables());
Assertions.assertEquals("select k1 from test_db.base_table",
statementContext.getOriginStatement().originStmt);
return null;
}
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,33 @@ public void testExecuteInternalRefreshPassesExcludedTriggerTables() throws Excep
Assertions.assertEquals(excluded, captured.get().getExcludedTriggerTables());
}

@Test
public void testIncrementalRefreshParsesSetVarWithItsStatementContext() throws Exception {
MTMV mtmv = mockMtmv();
Mockito.when(mtmv.getQuerySql()).thenReturn("SELECT /*+ SET_VAR(query_timeout=10) */ 1 AS k1");
Mockito.when(mtmv.getInsertedColumnNames()).thenReturn(List.of("k1"));
ConnectContext connectContext = new ConnectContext();
IvmIncrRefreshContext context = new IvmIncrRefreshContext(mtmv, connectContext, "audit",
queryId -> { }, null);
connectContext.setThreadLocalInfo();
try (MockedStatic<MTMVPlanUtil> mockedUtil = Mockito.mockStatic(MTMVPlanUtil.class)) {
mockedUtil.when(() -> MTMVPlanUtil.executeCommand(
Mockito.<ConnectContext>any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
.thenAnswer(inv -> {
StatementContext stmtCtx = inv.getArgument(2);
Assertions.assertSame(stmtCtx, connectContext.getStatementContext());
Assertions.assertEquals(mtmv.getQuerySql(), stmtCtx.getOriginStatement().originStmt);
Assertions.assertEquals(10, connectContext.getSessionVariable().getQueryTimeoutS());
return null;
});
new IvmIncrRefreshManager().executeInternalRefresh(context);
mockedUtil.verify(() -> MTMVPlanUtil.executeCommand(
Mockito.eq(connectContext), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()));
} finally {
ConnectContext.remove();
}
}

@Test
public void testManagerReturnsSuccessForEmptyBundles() throws Exception {
MTMV mtmv = mockMtmv();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,24 @@ public void testDryRunStatementContextCarriesExcludedTriggerTables() throws Exce
Assertions.assertEquals(excluded, stmtCtx.getExcludedTriggerTables());
}

@Test
public void testIncrementalDryRunParsesSetVarWithItsStatementContext() throws Exception {
RefreshMTMVInfo info = extractRefreshInfo("REFRESH MATERIALIZED VIEW db1.mv1 INCREMENTAL");
TestRefreshMTMVCommand command = new TestRefreshMTMVCommand(info, true);
MTMV mtmv = Mockito.mock(MTMV.class);
Mockito.when(mtmv.getQuerySql()).thenReturn("SELECT /*+ SET_VAR(query_timeout=10) */ 1 AS k1");
ConnectContext internalCtx = new ConnectContext();
internalCtx.setThreadLocalInfo();
try {
StatementContext stmtCtx = command.createDryRunStatementContext(mtmv, internalCtx);
new IvmIncrRefreshManager().buildQueryPlan(mtmv);
Assertions.assertSame(stmtCtx, internalCtx.getStatementContext());
Assertions.assertEquals(10, internalCtx.getSessionVariable().getQueryTimeoutS());
} finally {
ConnectContext.remove();
}
}

@Test
public void testIncrementalExplainCarriesExcludedTriggerTables() throws Exception {
RefreshMTMVInfo info = extractRefreshInfo("REFRESH MATERIALIZED VIEW db1.mv1 INCREMENTAL");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.apache.doris.planner.ExchangeNode;
import org.apache.doris.planner.OlapTableSink;
import org.apache.doris.planner.PlanFragment;
import org.apache.doris.qe.OriginStatement;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.thrift.TPartitionType;
import org.apache.doris.utframe.TestWithFeService;
Expand Down Expand Up @@ -241,6 +242,7 @@ void testPlannerKeepsIvmRowIdAsBigIntInSinkTupleAndOutputExprs() throws Exceptio
void testRunRefreshCommandExecutesIncrementalMtmv() throws Exception {
MTMV mtmv = getMtmv("ivm_mv");
StatementContext statementContext = createStatementCtx("refresh materialized view test.ivm_mv");
OriginStatement originStatement = statementContext.getOriginStatement();
statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.full(mtmv)));
UpdateMvByPartitionCommand command = newRefreshCommand(mtmv);
AtomicReference<StmtExecutor> executorRef = new AtomicReference<>();
Expand All @@ -260,12 +262,15 @@ void testRunRefreshCommandExecutesIncrementalMtmv() throws Exception {
executor.getContext().getStatementContext().getIvmRewriteContext().orElseThrow().getMode());
Assertions.assertSame(executor.getContext(), statementContext.getConnectContext());
Assertions.assertSame(statementContext, executor.getContext().getStatementContext());
Assertions.assertSame(originStatement, statementContext.getOriginStatement());
Assertions.assertSame(originStatement, executor.getParsedStmt().getOrigStmt());
}

@Test
void testExecuteCommandRebindsTaskStatementContextToExecutionContext() throws Exception {
MTMV mtmv = getMtmv("ivm_mv");
StatementContext statementContext = new StatementContext();
StatementContext statementContext = createStatementCtx("refresh materialized view test.ivm_mv");
OriginStatement originStatement = statementContext.getOriginStatement();
statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.full(mtmv)));
UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand.from(
mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext);
Expand All @@ -281,6 +286,8 @@ void testExecuteCommandRebindsTaskStatementContextToExecutionContext() throws Ex

Assertions.assertSame(executor.getContext(), statementContext.getConnectContext());
Assertions.assertSame(statementContext, executor.getContext().getStatementContext());
Assertions.assertSame(originStatement, statementContext.getOriginStatement());
Assertions.assertSame(originStatement, executor.getParsedStmt().getOrigStmt());
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ suite("test_ivm_refresh_dry_run") {
BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES ('replication_num' = '1')
AS SELECT k1, COUNT(*) AS cnt, SUM(v1) AS sum_v1
AS SELECT /*+ SET_VAR(query_timeout=180) */ k1, COUNT(*) AS cnt, SUM(v1) AS sum_v1
FROM test_ivm_refresh_dry_run_base
GROUP BY k1
"""
Expand Down
Loading