diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index dad9765cf9b594..7a97407c596baf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -658,6 +658,24 @@ public Optional getWorkloadGroup() { } } + /** + * The compute group explicitly declared on this MV, empty when the user did not declare one. + * An empty result keeps the existing implicit resolution (admin's group for auto refresh, + * the session's group for a manual REFRESH). + */ + public Optional getComputeGroup() { + readMvLock(); + try { + if (mvProperties.containsKey(PropertyAnalyzer.PROPERTIES_COMPUTE_GROUP) && !StringUtils + .isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_COMPUTE_GROUP))) { + return Optional.of(mvProperties.get(PropertyAnalyzer.PROPERTIES_COMPUTE_GROUP)); + } + return Optional.empty(); + } finally { + readMvUnlock(); + } + } + public boolean isUseForRewrite() { readMvLock(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudRoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudRoutineLoadManager.java index cee1a079424e98..e7257475441926 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudRoutineLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudRoutineLoadManager.java @@ -40,7 +40,12 @@ public class CloudRoutineLoadManager extends RoutineLoadManager { @Override public void addRoutineLoadJob(RoutineLoadJob routineLoadJob, String dbName, String tableName) throws UserException { - if (!Strings.isNullOrEmpty(ConnectContext.get().getCloudCluster())) { + // When the job declares a compute group explicitly, that declaration is the binding and the + // session's cluster must not overwrite it. + if (!Strings.isNullOrEmpty(routineLoadJob.getDeclaredComputeGroup())) { + LOG.info("routine load job {} pinned to declared compute group {}", + routineLoadJob.getName(), routineLoadJob.getDeclaredComputeGroup()); + } else if (!Strings.isNullOrEmpty(ConnectContext.get().getCloudCluster())) { routineLoadJob.setCloudCluster(ConnectContext.get().getCloudCluster()); } else { throw new UserException("cloud cluster is empty, please specify cloud cluster"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 746bb1d07a3e35..1e9470d2a45782 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -262,6 +262,7 @@ public class PropertyAnalyzer { "async_mv.query_rewrite.consistency_relaxed_tables"; public static final String PROPERTIES_REFRESH_PARTITION_NUM = "refresh_partition_num"; public static final String PROPERTIES_WORKLOAD_GROUP = "workload_group"; + public static final String PROPERTIES_COMPUTE_GROUP = "compute_group"; public static final String PROPERTIES_PARTITION_SYNC_LIMIT = "partition_sync_limit"; public static final String PROPERTIES_PARTITION_TIME_UNIT = "partition_sync_time_unit"; public static final String PROPERTIES_PARTITION_DATE_FORMAT = "partition_date_format"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 6de3a886beac6f..c6c3f134ca0ac8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -82,6 +82,7 @@ import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.resource.computegroup.ComputeGroupBindingUtil; import org.apache.doris.rpc.RpcException; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TCell; @@ -331,6 +332,10 @@ public void run() throws JobException { } mtmvSchemaChangeVersion = mtmv.getSchemaChangeVersion(); ConnectContext ctx = MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK); + // The planning done on this context (base table resolution, partition calculation) must see + // the same compute group the refresh will execute in, otherwise the workload group would be + // looked up in a different namespace. + applyComputeGroup(ctx); try { if (LOG.isDebugEnabled()) { String taskSessionContext = ctx.getSessionVariable().toJson().toJSONString(); @@ -1663,21 +1668,57 @@ private static void closeExecutionContext(ConnectContext executionContext) { } } - private void setupComputeGroup(ConnectContext ctx) { + /** + * Binds a refresh context to the compute group the refresh runs in, records that group on the task and + * re-checks the declared one. The declaration is read once, so the group that is checked is the group that + * is bound: an ALTER MATERIALIZED VIEW landing between two reads could otherwise have the check pass for + * the new group while this refresh runs in the old one. + */ + private void setupComputeGroup(ConnectContext ctx) throws UserException { if (!Config.isCloudMode()) { computeGroup = FeConstants.null_string; return; } - String taskComputeGroup = taskContext.getComputeGroup(); - if (!Strings.isNullOrEmpty(taskComputeGroup)) { - ctx.setCloudCluster(taskComputeGroup); - } + Optional declaredComputeGroup = applyComputeGroup(ctx); try { computeGroup = ctx.getCloudCluster(false); } catch (ComputeGroupException e) { computeGroup = FeConstants.null_string; LOG.warn("failed to resolve compute group for mtmv task, taskId: {}", getTaskId(), e); } + checkComputeGroupBeforeTask(ctx, declaredComputeGroup); + } + + // Returns the declaration it read, so that a re-check of the declared group checks the bound one. + private Optional applyComputeGroup(ConnectContext ctx) { + // A compute group declared on the MV pins every refresh, automatic or manual, to that group. + // Only when the MV declares nothing does a manual REFRESH keep borrowing the session's group, + // which is the behaviour every existing MV keeps. + Optional declaredComputeGroup = mtmv.getComputeGroup(); + String effective = declaredComputeGroup.orElse(taskContext.getComputeGroup()); + if (!Strings.isNullOrEmpty(effective)) { + // A no-op outside cloud mode. + ctx.setCloudCluster(effective); + } + return declaredComputeGroup; + } + + /** + * Re-checks the declared compute group before the refresh runs: it can be dropped and its + * privileges revoked while the MV exists, and without this the refresh would fail later with an + * unrelated message. + * + *

The identity used here is whatever the refresh actually runs as, which today is the + * hardcoded {@code admin} (see {@link MTMVPlanUtil#createBasicMvContext}). That makes the + * privilege half of the check always pass; the existence half is what has teeth right now. Once + * an MV carries a real owner, passing that owner here is the only change needed. + */ + private void checkComputeGroupBeforeTask(ConnectContext ctx, Optional declaredComputeGroup) + throws UserException { + // Only the explicitly declared compute group is re-checked; an MV that declares none borrows + // the session's group, which the user never chose for it. + ComputeGroupBindingUtil.checkComputeGroupBeforeTask(ctx.getCurrentUserIdentity(), + declaredComputeGroup.orElse(null)); } private void recordQueryId(String queryId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 4cd4434a73e6c3..6c644eb4792b5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -131,6 +131,7 @@ public abstract class RoutineLoadJob protected static final String STAR_STRING = "*"; public static final String WORKLOAD_GROUP = "workload_group"; + public static final String COMPUTE_GROUP = "compute_group"; @Getter @Setter @@ -438,6 +439,13 @@ protected void setOptional(CreateRoutineLoadInfo info) throws UserException { if (!StringUtils.isEmpty(info.getWorkloadGroupName())) { jobProperties.put(WORKLOAD_GROUP, info.getWorkloadGroupName()); } + + // Only write the key when the user declared it. An absent key means "not declared" and + // keeps the existing implicit resolution, which is also what later versions rely on to + // tell a pinned group apart from an inherited one. + if (!StringUtils.isEmpty(info.getComputeGroupName())) { + jobProperties.put(COMPUTE_GROUP, info.getComputeGroupName()); + } } protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { @@ -550,6 +558,11 @@ public String getWorkloadGroup() { return jobProperties.get(WORKLOAD_GROUP); } + // The compute group explicitly declared on the job, or null when the user did not declare one. + public String getDeclaredComputeGroup() { + return jobProperties.get(COMPUTE_GROUP); + } + public JobState getState() { return state; } @@ -797,7 +810,12 @@ public void setComment(String comment) { } public String getCloudCluster() { - return cloudCluster; + // An explicitly declared compute group wins over the cluster snapshotted from the session + // at create time. Keeping this in the getter (instead of writing both places) means every + // caller follows the declaration, and ALTER only has to update jobProperties, which is + // already covered by the existing edit log. + String declared = getDeclaredComputeGroup(); + return StringUtils.isEmpty(declared) ? cloudCluster : declared; } public int getSizeOfRoutineLoadTaskInfoList() { @@ -1059,13 +1077,16 @@ public TPipelineFragmentParams plan(NereidsStreamLoadPlanner planner, TUniqueId table.readLock(); try { if (Config.isCloudMode()) { + // Use the effective cluster, not the raw snapshot field: this context is what + // OlapTableSink uses to pick the backends the load writes to. + String effectiveCluster = getCloudCluster(); if (ConnectContext.get() == null) { ConnectContext ctx = new ConnectContext(); ctx.setThreadLocalInfo(); - ctx.setCloudCluster(cloudCluster); + ctx.setCloudCluster(effectiveCluster); needCleanCtx = true; } else { - ConnectContext.get().setCloudCluster(cloudCluster); + ConnectContext.get().setCloudCluster(effectiveCluster); } ConnectContext.get().setCurrentUserIdentity(this.getUserIdentity()); } else { @@ -1488,6 +1509,22 @@ public void updateState(JobState jobState, ErrorReason reason, boolean isReplay) .build()); } + // For failures found by refreshKafkaPartitions(), which update() runs before it takes the job lock. + // The state is checked again under the lock: a STOP that completed in between must not be turned + // back into PAUSED, as replay rejects STOPPED -> PAUSED. Only a NEED_SCHEDULE job is paused. A job that + // is still PAUSED, as during an auto resume attempt, keeps its reason (PAUSED -> PAUSED is rejected too), + // so a failure whose error code has to be recorded must not be reported from such an attempt. + protected void pauseIfNeedSchedule(ErrorReason reason) throws UserException { + writeLock(); + try { + if (state == JobState.NEED_SCHEDULE) { + unprotectUpdateState(JobState.PAUSED, reason, false /* not replay */); + } + } finally { + writeUnlock(); + } + } + protected void unprotectUpdateState(JobState jobState, ErrorReason reason, boolean isReplay) throws UserException { checkStateTransform(jobState); switch (jobState) { @@ -1667,7 +1704,9 @@ public void setCloudCluster(String cloudCluster) { } public String getClusterInfo() { - return Strings.nullToEmpty(cloudCluster); + // SHOW ROUTINE LOAD must display the cluster the job actually runs in, i.e. the declared + // compute group when there is one. + return Strings.nullToEmpty(getCloudCluster()); } // check the correctness of commit info @@ -1824,6 +1863,9 @@ public String getShowCreateInfo() { appendProperties(sb, JsonFileFormatProperties.PROP_JSON_ROOT, getJsonRoot(), false); appendProperties(sb, LoadCommand.STRICT_MODE, isStrictMode(), false); appendProperties(sb, LoadCommand.TIMEZONE, getTimezone(), false); + // Only a declared group, which follows ALTER. Writing the effective group instead would turn + // a group inherited from the creating session into a pinned one when the statement is replayed. + appendProperties(sb, CreateRoutineLoadInfo.COMPUTE_GROUP, getDeclaredComputeGroup(), false); appendProperties(sb, LoadCommand.EXEC_MEM_LIMIT, getMemLimit(), true); sb.append(")\n"); // 6. data_source @@ -2023,6 +2065,10 @@ public void gsonPostProcess() throws IOException { CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( origStmt.originStmt); CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); + // This re-parse only rebuilds the RoutineLoadDesc; it must not re-check resources + // that can legitimately have disappeared since the job was created, because the + // catch below turns any failure into the final CANCELLED state. + createRoutineLoadInfo.setReplay(true); // If tableId is set, resolve the current table name by ID so that // table rename / SWAP TABLE won't cause replay to fail with stale name in origStmt. if (!isMultiTable && tableId != 0) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java index e649a4edc7f2a0..c4a765cbc394d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java @@ -533,8 +533,9 @@ public long getAvailableBeForTask(long jobId, long previousBeId) throws UserExce if (availableBeIds.isEmpty()) { RoutineLoadJob job = getJob(jobId); if (job != null) { - String msg = "no available BE found for job " + jobId + ", cluster Name {}, " + job.getCloudCluster() - + "please check the BE status and user's cluster or tags"; + String msg = "no available BE found for job " + jobId + ", cluster name: " + + job.getCloudCluster() + + ", please check the BE status and user's cluster or tags"; job.updateState(RoutineLoadJob.JobState.PAUSED, new ErrorReason(InternalErrorCode.INTERNAL_ERR, msg), false /* not replay */); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java index c901cd41610abf..68bf066996e084 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java @@ -30,6 +30,7 @@ import org.apache.doris.common.util.LogKey; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.load.routineload.RoutineLoadJob.JobState; +import org.apache.doris.resource.computegroup.ComputeGroupBindingUtil; import org.apache.doris.system.Backend; import org.apache.doris.thrift.BackendService; import org.apache.doris.thrift.TNetworkAddress; @@ -135,9 +136,23 @@ private void scheduleOneTask(RoutineLoadTaskInfo routineLoadTaskInfo) throws Exc } try { - if (routineLoadManager.getJob(routineLoadTaskInfo.getJobId()).isFinal()) { + RoutineLoadJob job = routineLoadManager.getJob(routineLoadTaskInfo.getJobId()); + if (job.isFinal()) { return; } + + // The compute group the job declared can be dropped and the owner's USAGE on it revoked + // while the job keeps running, so it is re-checked before every task. + // + // This has to happen here rather than while the task is being built. Backend allocation + // below resolves the backends through that same compute group, so a missing group would + // otherwise pause the job with a generic "no available BE found" before this check was + // ever reached, which is exactly the misleading message it exists to replace. Running it + // first also keeps a failing check from leaving a transaction behind, because beginTxn + // has not happened yet. + ComputeGroupBindingUtil.checkComputeGroupBeforeTask(job.getUserIdentity(), + job.getDeclaredComputeGroup()); + // check if topic has more data to consume if (!routineLoadTaskInfo.hasMoreDataToConsume()) { needScheduleTasksQueue.addLast(routineLoadTaskInfo); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index 0570b356edf133..8477c6ed7ed261 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -57,6 +57,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.resource.computegroup.ComputeGroupBindingUtil; import org.apache.doris.rpc.RpcException; import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TFileCompressType; @@ -446,12 +447,32 @@ protected boolean refreshKafkaPartitions(boolean needAutoResume) throws UserExce if (customKafkaPartitions != null && !customKafkaPartitions.isEmpty()) { return true; } - return updateKafkaPartitions(); + return updateKafkaPartitions(needAutoResume); } return true; } - private boolean updateKafkaPartitions() throws UserException { + private boolean updateKafkaPartitions(boolean needAutoResume) throws UserException { + // The metadata request goes to the backends of the declared compute group, and after RESUME it runs + // before any task, so check the group here as RoutineLoadTaskScheduler does before each task. A + // dropped group would otherwise pause the job with "no alive backends" and the Kafka advice below. + // + // Not for an auto resume attempt, which is left to that check before the first task. The job is still + // PAUSED during the attempt and can not be paused again with another reason, so a failure here would + // keep its old, retryable one: a revoked USAGE would never become CANNOT_RESUME_ERR, and auto resume + // would retry for good. Once the metadata request below succeeds, the attempt moves the job to + // NEED_SCHEDULE, and the check before its first task records the failure. + if (!needAutoResume) { + try { + ComputeGroupBindingUtil.checkComputeGroupBeforeTask(getUserIdentity(), getDeclaredComputeGroup()); + } catch (UserException e) { + LOG.warn(new LogBuilder(LogKey.ROUTINE_LOAD_JOB, id) + .add("error_msg", e.getMessage()) + .build()); + pauseIfNeedSchedule(new ErrorReason(e.getErrorCode(), e.getMessage())); + return false; + } + } try { this.newCurrentKafkaPartition = getAllKafkaPartitions(); } catch (Exception e) { @@ -461,11 +482,7 @@ private boolean updateKafkaPartitions() throws UserException { LOG.warn(new LogBuilder(LogKey.ROUTINE_LOAD_JOB, id) .add("error_msg", msg) .build(), e); - if (this.state == JobState.NEED_SCHEDULE) { - unprotectUpdateState(JobState.PAUSED, - new ErrorReason(InternalErrorCode.PARTITIONS_ERR, msg), - false /* not replay */); - } + pauseIfNeedSchedule(new ErrorReason(InternalErrorCode.PARTITIONS_ERR, msg)); return false; } return true; @@ -779,9 +796,26 @@ public Map getCustomProperties() { public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); KafkaDataSourceProperties dataSourceProperties = (KafkaDataSourceProperties) command.getDataSourceProperties(); + String declaredComputeGroup = jobProperties.get(CreateRoutineLoadInfo.COMPUTE_GROUP); + if (!Strings.isNullOrEmpty(declaredComputeGroup)) { + // Every task looks the job's workload group up in the compute group the job runs in + // (KafkaTaskInfo#rePlan), so moving the job checks it in the new group, as CREATE does in the group + // it declares. One the new group does not have would otherwise fail every task with the retryable + // CREATE_TASKS_ERR, which auto resume repeats for good. Checked as the owner, whom the tasks run as. + String workloadGroup = jobProperties.containsKey(CreateRoutineLoadInfo.WORKLOAD_GROUP) + ? jobProperties.get(CreateRoutineLoadInfo.WORKLOAD_GROUP) : getWorkloadGroup(); + if (!Strings.isNullOrEmpty(workloadGroup)) { + CreateRoutineLoadInfo.checkWorkloadGroup(getUserIdentity(), workloadGroup, declaredComputeGroup); + } + } if (null != dataSourceProperties) { + // The offsets are converted before modifyPropertiesInternal installs the new properties, + // so a compute group this ALTER declares has to be used already: for + // ALTER ... PROPERTIES("compute_group" = "cg_b") FROM KAFKA("kafka_offsets" = "OFFSET_END") + // the job's current group may be gone, which is why it is being moved to cg_b. // if the partition offset is set by timestamp, convert it to real offset - convertOffset(dataSourceProperties); + convertOffset(dataSourceProperties, Strings.isNullOrEmpty(declaredComputeGroup) + ? getComputeGroupName() : declaredComputeGroup); } writeLock(); @@ -800,12 +834,12 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } } - private void convertOffset(KafkaDataSourceProperties dataSourceProperties) throws UserException { + private void convertOffset(KafkaDataSourceProperties dataSourceProperties, String computeGroupName) + throws UserException { List> partitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); if (partitionOffsets.isEmpty()) { return; } - String computeGroupName = getComputeGroupName(); List> newOffsets; if (dataSourceProperties.isOffsetsForTimes()) { newOffsets = KafkaUtil.getOffsetsForTimes( diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaTaskInfo.java index 322a38151c6add..b911a868865dcd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaTaskInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaTaskInfo.java @@ -87,6 +87,10 @@ public List getPartitions() { public TRoutineLoadTask createRoutineLoadTask() throws UserException { KafkaRoutineLoadJob routineLoadJob = (KafkaRoutineLoadJob) routineLoadManager.getJob(jobId); + // The declared compute group is re-checked before every task, but that happens earlier, in + // RoutineLoadTaskScheduler#scheduleOneTask: it has to run before backend allocation and + // before beginTxn, neither of which has happened by the time this method is called. + // init tRoutineLoadTask and create plan fragment TRoutineLoadTask tRoutineLoadTask = new TRoutineLoadTask(); TUniqueId queryId = new TUniqueId(id.getMostSignificantBits(), id.getLeastSignificantBits()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 7cebc3f5165b49..2a9e566a80de62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -366,11 +366,7 @@ private boolean updateKinesisShards() throws UserException { LOG.warn(new LogBuilder(LogKey.ROUTINE_LOAD_JOB, id) .add("error_msg", msg) .build(), e); - if (this.state == JobState.NEED_SCHEDULE) { - unprotectUpdateState(JobState.PAUSED, - new ErrorReason(InternalErrorCode.PARTITIONS_ERR, msg), - false /* not replay */); - } + pauseIfNeedSchedule(new ErrorReason(InternalErrorCode.PARTITIONS_ERR, msg)); return false; } return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java index 1d2b3a9daae794..01458d4cac539e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java @@ -26,10 +26,12 @@ import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.catalog.info.TableNameInfo; +import org.apache.doris.common.UserException; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.resource.computegroup.ComputeGroupBindingUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -57,6 +59,7 @@ public class MTMVPropertyUtil { PropertyAnalyzer.ASYNC_MV_QUERY_REWRITE_CONSISTENCY_RELAXED_TABLES, PropertyAnalyzer.PROPERTIES_REFRESH_PARTITION_NUM, PropertyAnalyzer.PROPERTIES_WORKLOAD_GROUP, + PropertyAnalyzer.PROPERTIES_COMPUTE_GROUP, PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT, PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT, PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT, @@ -84,6 +87,9 @@ public static void analyzeProperty(String key, String value) { case PropertyAnalyzer.PROPERTIES_WORKLOAD_GROUP: analyzeWorkloadGroup(value); break; + case PropertyAnalyzer.PROPERTIES_COMPUTE_GROUP: + analyzeComputeGroup(value); + break; case PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT: analyzePartitionTimeUnit(value); break; @@ -140,6 +146,17 @@ private static void analyzePartitionTimeUnit(String value) { } } + private static void analyzeComputeGroup(String value) { + if (StringUtils.isEmpty(value)) { + return; + } + try { + ComputeGroupBindingUtil.validateDeclaredComputeGroup(ConnectContext.get(), value); + } catch (UserException e) { + throw new AnalysisException(e.getMessage(), e); + } + } + private static void analyzeWorkloadGroup(String value) { if (StringUtils.isEmpty(value)) { return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index 367480c5d934d9..3473a5e3fd35a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -40,6 +40,7 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.resource.computegroup.ComputeGroupBindingUtil; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; @@ -74,6 +75,7 @@ public class AlterRoutineLoadCommand extends AlterCommand { .add(CreateRoutineLoadInfo.STRICT_MODE) .add(CreateRoutineLoadInfo.TIMEZONE) .add(CreateRoutineLoadInfo.WORKLOAD_GROUP) + .add(CreateRoutineLoadInfo.COMPUTE_GROUP) .add(JsonFileFormatProperties.PROP_JSON_PATHS) .add(JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY) .add(JsonFileFormatProperties.PROP_NUM_AS_STRING) @@ -162,7 +164,7 @@ public void validate(ConnectContext ctx) throws UserException { labelNameInfo.validate(ctx); FeNameFormat.checkCommonName(NAME_TYPE, labelNameInfo.getLabel()); // check routine load job properties include desired concurrent number etc. - checkJobProperties(); + checkJobProperties(ctx); // check load properties RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() .getJob(getDbName(), getJobName()); @@ -177,7 +179,7 @@ public void validate(ConnectContext ctx) throws UserException { } } - private void checkJobProperties() throws UserException { + private void checkJobProperties(ConnectContext ctx) throws UserException { Optional optional = jobProperties.keySet().stream().filter( entity -> !CONFIGURABLE_JOB_PROPERTIES_SET.contains(entity)).findFirst(); if (optional.isPresent()) { @@ -287,6 +289,19 @@ private void checkJobProperties() throws UserException { analyzedJobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, modeStr.toUpperCase()); } + if (jobProperties.containsKey(CreateRoutineLoadInfo.COMPUTE_GROUP)) { + String computeGroup = jobProperties.get(CreateRoutineLoadInfo.COMPUTE_GROUP); + if (!StringUtil.isEmpty(computeGroup)) { + // Validated right here instead of letting a bad name pause the whole job on the next task. + // The job's workload group is then checked in this group by KafkaRoutineLoadJob#modifyProperties, + // which also knows the one the job keeps when this ALTER sets none. + ComputeGroupBindingUtil.validateDeclaredComputeGroup(ctx, computeGroup); + // a combined ALTER converts its new Kafka offsets in this group, see KafkaRoutineLoadJob + ComputeGroupBindingUtil.activateDeclaredComputeGroup(computeGroup); + analyzedJobProperties.put(CreateRoutineLoadInfo.COMPUTE_GROUP, computeGroup); + } + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.WORKLOAD_GROUP)) { String workloadGroup = jobProperties.get(CreateRoutineLoadInfo.WORKLOAD_GROUP); if (!StringUtil.isEmpty(workloadGroup)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java index 31d7296fae0db0..3a3518282935af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java @@ -26,6 +26,7 @@ import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.UserException; import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import org.apache.doris.mtmv.MTMVPlanUtil; @@ -45,6 +46,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.resource.computegroup.ComputeGroupBindingUtil; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; @@ -111,14 +113,13 @@ private void dryRunRefresh(ConnectContext ctx, StmtExecutor executor) throws Exc + "only supports IVM materialized views"); } - // createMTMVContext installs the internal ADMIN context as the thread-local - // ConnectContext. Restore the caller's context afterwards, mirroring ExplainCommand: - // the next statement of the same multi-statement request must not observe the - // internal identity/session. + // createDryRunConnectContext installs the internal ADMIN context as the thread-local + // ConnectContext, and leaves it installed when its compute group check fails. Restore the + // caller's context afterwards, mirroring ExplainCommand: the next statement of the same + // multi-statement request must not observe the internal identity/session. ConnectContext previousCtx = ConnectContext.get(); try { - ConnectContext internalCtx = MTMVPlanUtil.createMTMVContext( - mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK); + ConnectContext internalCtx = createDryRunConnectContext(mtmv); StatementContext stmtCtx = createDryRunStatementContext(mtmv, internalCtx); LogicalPlan queryPlan = new IvmIncrRefreshManager().buildQueryPlan(mtmv); @@ -145,6 +146,26 @@ private void dryRunRefresh(ConnectContext ctx, StmtExecutor executor) throws Exc } } + /** + * The internal context the dry run plans and executes its delta query on. Like the refresh + * contexts of MTMVTask, it is bound to the compute group the MV declares, which is re-checked + * because it can have been dropped since CREATE / ALTER, so the delta runs, and its workload + * group is looked up, in the group every refresh of that MV runs in. An MV that declares none + * keeps the existing resolution. + */ + @VisibleForTesting + ConnectContext createDryRunConnectContext(MTMV mtmv) throws UserException { + ConnectContext internalCtx = MTMVPlanUtil.createMTMVContext( + mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK); + // Read once, so the group that is checked is the group that is bound. + Optional declaredComputeGroup = mtmv.getComputeGroup(); + // A no-op outside cloud mode. + declaredComputeGroup.ifPresent(internalCtx::setCloudCluster); + ComputeGroupBindingUtil.checkComputeGroupBeforeTask(internalCtx.getCurrentUserIdentity(), + declaredComputeGroup.orElse(null)); + return internalCtx; + } + @VisibleForTesting StatementContext createDryRunStatementContext(MTMV mtmv, ConnectContext internalCtx) { StatementContext stmtCtx = new StatementContext( @@ -178,7 +199,12 @@ public Plan getExplainPlan(ConnectContext ctx) throws Exception { } protected ConnectContext createExplainConnectContext(MTMV mtmv) { - return MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK); + ConnectContext explainCtx = MTMVPlanUtil.createMTMVContext( + mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK); + // Plan, and pick backends, in the group the MV declares, as MTMVTask does for its planning + // context. Nothing is executed here, so the group is not re-checked. + mtmv.getComputeGroup().ifPresent(explainCtx::setCloudCluster); + return explainCtx; } protected LogicalPlan createRefreshCommand(MTMV mtmv, StatementContext statementContext) throws Exception { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java index e8a75c0299b4d6..94ed9345356921 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; +import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; @@ -51,6 +52,7 @@ import org.apache.doris.nereids.trees.plans.commands.load.LoadWhereClause; import org.apache.doris.nereids.util.PlanUtils; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.resource.computegroup.ComputeGroupBindingUtil; import org.apache.doris.resource.workloadgroup.WorkloadGroup; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; import org.apache.doris.thrift.TUniqueKeyUpdateMode; @@ -89,6 +91,7 @@ public class CreateRoutineLoadInfo { public static final String PARTIAL_UPDATE_NEW_KEY_POLICY = "partial_update_new_key_behavior"; public static final String UNIQUE_KEY_UPDATE_MODE = "unique_key_update_mode"; public static final String WORKLOAD_GROUP = "workload_group"; + public static final String COMPUTE_GROUP = "compute_group"; public static final String ENDPOINT_REGEX = "[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]"; public static final String SEND_BATCH_PARALLELISM = "send_batch_parallelism"; public static final String LOAD_TO_SINGLE_TABLET = "load_to_single_tablet"; @@ -125,6 +128,7 @@ public class CreateRoutineLoadInfo { .add(PARTIAL_UPDATE_NEW_KEY_POLICY) .add(UNIQUE_KEY_UPDATE_MODE) .add(WORKLOAD_GROUP) + .add(COMPUTE_GROUP) .add(FileFormatProperties.PROP_FORMAT) .add(JsonFileFormatProperties.PROP_JSON_PATHS) .add(JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY) @@ -164,6 +168,20 @@ public class CreateRoutineLoadInfo { private String workloadGroupName; + private String computeGroupName; + + // Set only by the metadata load path (RoutineLoadJob#gsonPostProcess), which re-parses the + // stored statement to rebuild the RoutineLoadDesc rather than to admit a new job. Resource + // existence must not be re-checked there, neither the compute group nor the workload group: + // an exception during metadata load is turned into JobState.CANCELLED, which is final and + // cannot be undone by RESUME, and both lookups can fail for a job that is fine. A compute group + // can be dropped, renamed or scaled to zero backends while a job exists, and the stored + // statement is the CREATE one, so a job ALTERed from cg_a to cg_b still names cg_a there. And + // an image load reads the routineLoadJobs module before workloadGroups, so no workload group + // exists yet when the job is re-parsed. A missing group has to pause the job through the per + // task check instead, which is recoverable. + private boolean isReplay = false; + /** * support partial columns load(Only Unique Key Columns) */ @@ -266,6 +284,25 @@ public static TUniqueKeyUpdateMode parseAndValidateUniqueKeyUpdateMode(String mo return mode; } + /** + * Checks a routine load job's workload group the way its tasks resolve it ({@code KafkaTaskInfo#rePlan}): + * as {@code user}, and in cloud mode in the namespace of {@code computeGroup}, the group the job runs in. A + * workload group lives in the namespace of a compute group: the same name under a different compute group + * is a different group. {@code computeGroup} is not used outside cloud mode. + */ + public static void checkWorkloadGroup(UserIdentity user, String workloadGroup, String computeGroup) + throws UserException { + ConnectContext tmpCtx = new ConnectContext(); + tmpCtx.setCurrentUserIdentity(user); + tmpCtx.getSessionVariable().setWorkloadGroup(workloadGroup); + // a no-op outside cloud mode + tmpCtx.setCloudCluster(computeGroup); + List wgList = Env.getCurrentEnv().getWorkloadGroupMgr().getWorkloadGroup(tmpCtx); + if (wgList.size() == 0) { + throw new UserException("Can not find workload group " + workloadGroup); + } + } + public String getName() { return name; } @@ -378,6 +415,19 @@ public String getWorkloadGroupName() { return workloadGroupName; } + public String getComputeGroupName() { + return computeGroupName; + } + + /** + * Marks this info object as rebuilt from persisted metadata instead of parsed from a user + * statement, so that {@link #validate(ConnectContext)} skips checks against resources that may + * legitimately have disappeared since the job was created. + */ + public void setReplay(boolean isReplay) { + this.isReplay = isReplay; + } + /** * analyze create table info */ @@ -599,18 +649,35 @@ public void checkJobProperties() throws UserException { RoutineLoadJob.DEFAULT_LOAD_TO_SINGLE_TABLET, LOAD_TO_SINGLE_TABLET + " should be a boolean"); + String inputComputeGroupStr = jobProperties.get(COMPUTE_GROUP); + if (!StringUtils.isEmpty(inputComputeGroupStr)) { + // The name is always adopted, because the workload group check below resolves in its + // namespace, but it is only validated when a user is actually declaring it. On the + // metadata load path the declared group may have been dropped long ago, and failing + // there would cancel the job permanently instead of pausing it - see isReplay. + if (!isReplay) { + ComputeGroupBindingUtil.validateDeclaredComputeGroup(ConnectContext.get(), inputComputeGroupStr); + // Kafka metadata requests go through this group as soon as the job is created (custom + // partitions or offsets), so start it now if it is suspended, as USE @ would. + ComputeGroupBindingUtil.activateDeclaredComputeGroup(inputComputeGroupStr); + } + this.computeGroupName = inputComputeGroupStr; + } + String inputWorkloadGroupStr = jobProperties.get(WORKLOAD_GROUP); if (!StringUtils.isEmpty(inputWorkloadGroupStr)) { - ConnectContext tmpCtx = new ConnectContext(); - tmpCtx.setCurrentUserIdentity(ConnectContext.get().getCurrentUserIdentity()); - tmpCtx.getSessionVariable().setWorkloadGroup(inputWorkloadGroupStr); - if (Config.isCloudMode()) { - tmpCtx.setCloudCluster(ConnectContext.get().getCloudCluster()); - } - List wgList = Env.getCurrentEnv().getWorkloadGroupMgr() - .getWorkloadGroup(tmpCtx); - if (wgList.size() == 0) { - throw new UserException("Can not find workload group " + inputWorkloadGroupStr); + // Like the compute group above, the name is adopted on the metadata load path without + // being resolved, see isReplay. For example, a job created with ("compute_group" = "cg_a", + // "workload_group" = "wg_a") and moved to cg_b was cancelled by the first restart after + // cg_a was dropped, because this lookup resolved wg_a in cg_a. + if (!isReplay) { + // Checked in the compute group the job will run in: the declared one, or else the session's, + // which the job snapshots when it is created. + String computeGroup = this.computeGroupName; + if (Config.isCloudMode() && StringUtils.isEmpty(computeGroup)) { + computeGroup = ConnectContext.get().getCloudCluster(); + } + checkWorkloadGroup(ConnectContext.get().getCurrentUserIdentity(), inputWorkloadGroupStr, computeGroup); } this.workloadGroupName = inputWorkloadGroupStr; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 8d93e63de81941..c1255182a29ede 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -1558,6 +1558,11 @@ public boolean supportHandleByFe() { } public void setCloudCluster(String cluster) { + // A compute group only exists in cloud mode. Swallowing the call here instead of making + // every caller wrap it in `if (Config.isCloudMode())` keeps that check in one place. + if (!Config.isCloudMode()) { + return; + } this.getSessionVariable().setCloudCluster(cluster); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/computegroup/ComputeGroupBindingUtil.java b/fe/fe-core/src/main/java/org/apache/doris/resource/computegroup/ComputeGroupBindingUtil.java new file mode 100644 index 00000000000000..8fd765ba1b3f44 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/computegroup/ComputeGroupBindingUtil.java @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.resource.computegroup; + +import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Config; +import org.apache.doris.common.InternalErrorCode; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import org.apache.commons.lang3.StringUtils; + +/** + * Validation for the {@code compute_group} property that can be declared on background jobs + * (routine load / async materialized view). + * + *

This is a transitional binding: it adds the ability to declare a compute group, and + * re-checks that declaration before every task, but does not change how the group is resolved for + * jobs that declare none. The property name and its value space are intentionally identical to the + * final {@code (owner, compute_group, workload_group)} design, so that metadata written by this + * version can be read as an explicit "pin" by later versions without any conversion. + * + *

Two values are rejected on purpose: + *

    + *
  • Any value in non-cloud mode - non-cloud support is not part of this transitional change, + * so no non-cloud metadata will ever carry this key.
  • + *
  • {@code DEFAULT} (case insensitive) - it is reserved by the final design to mean + * "follow the owner's default group at runtime". Allowing a job to pin a group literally + * named {@code DEFAULT} would silently change its behavior after upgrading.
  • + *
+ */ +public class ComputeGroupBindingUtil { + + /** + * Reserved value in the final binding design: "not pinned, follow the owner's default group". + * Rejected here so that no job can pin a group literally named {@code DEFAULT}. + */ + public static final String RESERVED_DEFAULT = "DEFAULT"; + + public static final String ERR_NON_CLOUD = + "Property 'compute_group' is only supported in cloud mode for now."; + + private ComputeGroupBindingUtil() { + } + + /** + * Validates a user declared compute group name. + * + *

An empty value means "not declared" and is treated as a no-op by the caller, which must + * not write the key into the job's property map at all. + * + * @param ctx the context of the user executing CREATE / ALTER; privileges are checked against + * this user, matching how {@code workload_group} is validated today + * @param computeGroup the declared name + */ + public static void validateDeclaredComputeGroup(ConnectContext ctx, String computeGroup) throws UserException { + if (StringUtils.isEmpty(computeGroup)) { + return; + } + + if (!Config.isCloudMode()) { + throw new UserException(ERR_NON_CLOUD); + } + + if (RESERVED_DEFAULT.equalsIgnoreCase(computeGroup)) { + throw new UserException("'" + RESERVED_DEFAULT + "' is a reserved value for property 'compute_group'" + + " and can not be used as a compute group name here."); + } + + if (ctx == null) { + throw new UserException("Can not validate property 'compute_group' without a connect context."); + } + + // Same two checks, and the same order, as `USE @`. + if (!Env.getCurrentEnv().getAccessManager().checkCloudPriv(ctx.getCurrentUserIdentity(), + computeGroup, PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)) { + throw new UserException("USAGE denied to user '" + ctx.getQualifiedUser() + + "' for compute group '" + computeGroup + "'"); + } + + if (!((CloudSystemInfoService) Env.getCurrentSystemInfo()).getCloudClusterNames().contains(computeGroup)) { + throw new UserException("Compute group '" + computeGroup + "' not found."); + } + } + + /** + * Starts a declared compute group that is suspended, the third step {@code USE @} + * takes after the two checks above ({@code CloudEnv#changeCloudCluster}); a no-op when auto start + * is disabled or the group is running. + * + *

Routine load needs it: CREATE and ALTER send their Kafka metadata requests (custom + * partitions, offset conversion) through the declared group's backends while the statement + * runs. For example, a session on cg_a running {@code CREATE ROUTINE LOAD ... + * PROPERTIES("compute_group" = "cg_b") FROM KAFKA("kafka_partitions" = "0", ...)} while cg_b is + * suspended found no live backend in cg_b and failed, since nothing else starts cg_b. It must + * be called on the statement thread before any job lock, and it does not change the session's + * group. It is not part of {@link #validateDeclaredComputeGroup}: an async MV sends nothing to + * its group at CREATE, and its refresh resolves backends through the session group it binds + * ({@code CloudSystemInfoService#getBackendsByCurrentCluster}), which starts the group then. + */ + public static void activateDeclaredComputeGroup(String computeGroup) throws UserException { + ((CloudSystemInfoService) Env.getCurrentSystemInfo()).waitForAutoStart(computeGroup); + } + + /** + * Re-checks the compute group a job declared, before each of its tasks runs. + * + *

Creation-time validation alone is not enough: the group can be dropped and the owner's + * privileges can be revoked while the job keeps running, and without this check the task would + * silently keep using a group its owner is no longer entitled to, or fail much later with an + * unrelated message such as "no available BE found". + * + *

Everything is checked against {@code owner}, the identity the task actually runs as, not + * against whoever created or last altered the job. + * + *

The workload group is deliberately out of scope here. Both callers resolve it a little + * later through {@code WorkloadGroupMgr#getWorkloadGroup(ConnectContext)} - routine load in + * {@code KafkaTaskInfo#createRoutineLoadTask}, an MV refresh in the coordinator - and that + * already runs the same USAGE check against the same owner and the same existence check in the + * same compute group namespace. + * + * @param owner the identity the task runs as + * @param computeGroup the compute group declared on the job; empty means the job declared none + * and there is nothing to re-check + */ + public static void checkComputeGroupBeforeTask(UserIdentity owner, String computeGroup) + throws UserException { + if (owner == null) { + // Jobs created before the owner was persisted; nothing to check them against. + return; + } + + if (!Config.isCloudMode() || StringUtils.isEmpty(computeGroup)) { + return; + } + + // Deliberately not ComputeGroupMgr.getComputeGroupByName() for the existence check: when + // the group is missing that builds a hint message from the thread-local ConnectContext, + // and the callers here are background threads that do not have one. + // + // A missing group is left on the default INTERNAL_ERR, which RoutineLoadTaskScheduler + // reports as a retryable pause: the name can come back on its own, because a compute group + // that is merely scaled to zero backends is removed from the cluster map and re-added when + // it scales up again. Auto resume then picks the job up without an operator. + if (!((CloudSystemInfoService) Env.getCurrentSystemInfo()).getCloudClusterNames() + .contains(computeGroup)) { + throw new UserException("Compute group '" + computeGroup + "' not found."); + } + + // A revoked privilege is the opposite: somebody decided this owner may no longer use this + // group, and nothing will undo that by itself. CANNOT_RESUME_ERR keeps ScheduleRule from + // auto resuming the job, which would otherwise pause and resume it every few minutes for + // as long as the grant is missing. That only works where the failure pauses the job, so + // routine load runs this for a job about to be scheduled and not for one that is still + // PAUSED: an auto resume attempt could not replace that job's old, retryable reason (see + // KafkaRoutineLoadJob#updateKafkaPartitions). + if (!Env.getCurrentEnv().getAccessManager().checkCloudPriv(owner, computeGroup, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)) { + throw new UserException(InternalErrorCode.CANNOT_RESUME_ERR, + "USAGE denied to user '" + owner.getQualifiedUser() + + "' for compute group '" + computeGroup + "'"); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadComputeGroupTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadComputeGroupTest.java new file mode 100644 index 00000000000000..e167da1bc8aea6 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadComputeGroupTest.java @@ -0,0 +1,964 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.load.routineload; + +import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.load.CloudRoutineLoadManager; +import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.InternalErrorCode; +import org.apache.doris.common.LoadException; +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.kafka.KafkaUtil; +import org.apache.doris.datasource.kinesis.KinesisUtil; +import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; +import org.apache.doris.load.routineload.kafka.KafkaProgress; +import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; +import org.apache.doris.load.routineload.kafka.KafkaTaskInfo; +import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; +import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; +import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.persist.meta.PersistMetaModules; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.resource.computegroup.ComputeGroupMgr; +import org.apache.doris.resource.workloadgroup.WorkloadGroup; +import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.LockSupport; +import java.util.stream.Collectors; + +/** + * The transitional {@code compute_group} declaration on routine load jobs. + * + *

The declaration lives in {@code jobProperties} under the key {@code compute_group}. It is only + * written when the user actually declared one, so an absent key keeps the old behaviour of using the + * cluster snapshotted from the creating session, and later versions can tell a pinned group apart + * from an inherited one just by looking at whether the key is present. + */ +public class RoutineLoadComputeGroupTest { + + private static final String SESSION_SNAPSHOT_CG = "cg_from_session"; + private static final String DECLARED_CG = "cg_declared"; + + private String originalDeployMode; + private String originalCloudUniqueId; + private SystemInfoService originalSystemInfo; + private AccessControllerManager originalAccessManager; + private WorkloadGroupMgr originalWorkloadGroupMgr; + private ComputeGroupMgr originalComputeGroupMgr; + + @BeforeEach + public void setUp() { + originalDeployMode = Config.deploy_mode; + originalCloudUniqueId = Config.cloud_unique_id; + originalSystemInfo = Env.getCurrentSystemInfo(); + originalAccessManager = Env.getCurrentEnv().getAccessManager(); + originalWorkloadGroupMgr = Env.getCurrentEnv().getWorkloadGroupMgr(); + originalComputeGroupMgr = Env.getCurrentEnv().getComputeGroupMgr(); + } + + @AfterEach + public void tearDown() { + Config.deploy_mode = originalDeployMode; + Config.cloud_unique_id = originalCloudUniqueId; + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", originalSystemInfo); + Deencapsulation.setField(Env.getCurrentEnv(), "accessManager", originalAccessManager); + Deencapsulation.setField(Env.getCurrentEnv(), "workloadGroupMgr", originalWorkloadGroupMgr); + Deencapsulation.setField(Env.getCurrentEnv(), "computeGroupMgr", originalComputeGroupMgr); + ConnectContext.remove(); + } + + /** + * Puts the FE into cloud mode with a known set of existing compute groups, and a session that + * holds (or does not hold) USAGE on whatever it asks for. + */ + private void enterCloudMode(boolean hasPriv, List existingClusters) { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + + mockAccessManager(hasPriv); + + Map> clusterToBackends = Maps.newHashMap(); + for (String cluster : existingClusters) { + clusterToBackends.put(cluster, Lists.newArrayList(createBackend(10000L + clusterToBackends.size()))); + } + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", + new ClusterAwareCloudSystemInfoService(clusterToBackends)); + + ConnectContext ctx = new ConnectContext(); + ctx.setCurrentUserIdentity(UserIdentity.ADMIN); + ctx.setThreadLocalInfo(); + } + + // USAGE on every compute group and workload group the session asks for, or on none. + private void mockAccessManager(boolean hasPriv) { + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.when(accessManager.checkCloudPriv(Mockito.any(UserIdentity.class), Mockito.anyString(), + Mockito.any(PrivPredicate.class), Mockito.any(ResourceTypeEnum.class))).thenReturn(hasPriv); + Mockito.when(accessManager.checkWorkloadGroupPriv(Mockito.any(ConnectContext.class), Mockito.anyString(), + Mockito.any(PrivPredicate.class))).thenReturn(hasPriv); + Deencapsulation.setField(Env.getCurrentEnv(), "accessManager", accessManager); + } + + private ClusterAwareCloudSystemInfoService cloudSystemInfo() { + return (ClusterAwareCloudSystemInfoService) Env.getCurrentSystemInfo(); + } + + private KafkaRoutineLoadJob newJob(String snapshotCluster, String declaredComputeGroup) { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(); + // serialization walks this field, give it something non-null + job.setOrigStmt(new OriginStatement("CREATE ROUTINE LOAD test ON tbl", 0)); + Deencapsulation.setField(job, "cloudCluster", snapshotCluster); + if (declaredComputeGroup != null) { + Map jobProperties = Maps.newHashMap(); + jobProperties.put(RoutineLoadJob.COMPUTE_GROUP, declaredComputeGroup); + Deencapsulation.setField(job, "jobProperties", jobProperties); + } + return job; + } + + @Test + public void testDeclaredComputeGroupWinsOverSessionSnapshot() { + KafkaRoutineLoadJob job = newJob(SESSION_SNAPSHOT_CG, DECLARED_CG); + Assertions.assertEquals(DECLARED_CG, job.getDeclaredComputeGroup()); + Assertions.assertEquals(DECLARED_CG, job.getCloudCluster()); + } + + // No declaration must leave the existing behaviour completely untouched. + @Test + public void testFallsBackToSessionSnapshotWhenNotDeclared() { + KafkaRoutineLoadJob job = newJob(SESSION_SNAPSHOT_CG, null); + Assertions.assertNull(job.getDeclaredComputeGroup()); + Assertions.assertEquals(SESSION_SNAPSHOT_CG, job.getCloudCluster()); + } + + // An empty declared value is treated as "not declared", same as the workload group property. + @Test + public void testEmptyDeclarationFallsBackToSnapshot() { + KafkaRoutineLoadJob job = newJob(SESSION_SNAPSHOT_CG, ""); + Assertions.assertEquals(SESSION_SNAPSHOT_CG, job.getCloudCluster()); + } + + // SHOW ROUTINE LOAD must report the cluster the job actually runs in. + @Test + public void testClusterInfoShowsEffectiveComputeGroup() { + Assertions.assertEquals(DECLARED_CG, newJob(SESSION_SNAPSHOT_CG, DECLARED_CG).getClusterInfo()); + Assertions.assertEquals(SESSION_SNAPSHOT_CG, newJob(SESSION_SNAPSHOT_CG, null).getClusterInfo()); + Assertions.assertEquals("", newJob(null, null).getClusterInfo()); + } + + // Metadata compatibility: the declaration is carried in the already existing jobProperties map, + // so it survives a serialization round trip without any new persisted field. + @Test + public void testMetadataRoundTripKeepsDeclaration() { + KafkaRoutineLoadJob job = newJob(SESSION_SNAPSHOT_CG, DECLARED_CG); + String json = GsonUtils.GSON.toJson(job); + Assertions.assertTrue(json.contains(RoutineLoadJob.COMPUTE_GROUP), json); + + KafkaRoutineLoadJob restored = GsonUtils.GSON.fromJson(json, KafkaRoutineLoadJob.class); + Assertions.assertEquals(DECLARED_CG, restored.getDeclaredComputeGroup()); + Assertions.assertEquals(DECLARED_CG, restored.getCloudCluster()); + } + + // Downgrade safety: metadata written without the key must still load and behave as before. + @Test + public void testMetadataRoundTripWithoutDeclaration() { + KafkaRoutineLoadJob job = newJob(SESSION_SNAPSHOT_CG, null); + String json = GsonUtils.GSON.toJson(job); + Assertions.assertFalse(json.contains(RoutineLoadJob.COMPUTE_GROUP), json); + + KafkaRoutineLoadJob restored = GsonUtils.GSON.fromJson(json, KafkaRoutineLoadJob.class); + Assertions.assertNull(restored.getDeclaredComputeGroup()); + Assertions.assertEquals(SESSION_SNAPSHOT_CG, restored.getCloudCluster()); + } + + /** + * Isolation: two jobs pinned to different compute groups must be scheduled onto disjoint sets of + * backends, so neither can consume the other's resources. The backend set is resolved per job + * and on every task allocation, not once at create time. + */ + @Test + public void testJobsWithDifferentComputeGroupsGetDisjointBackends() throws LoadException { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + + Backend beInA1 = createBackend(10001L); + Backend beInA2 = createBackend(10002L); + Backend beInB = createBackend(10003L); + Map> clusterToBackends = Maps.newHashMap(); + clusterToBackends.put("cg_a", Lists.newArrayList(beInA1, beInA2)); + clusterToBackends.put("cg_b", Lists.newArrayList(beInB)); + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", + new ClusterAwareCloudSystemInfoService(clusterToBackends)); + + KafkaRoutineLoadJob jobA = newJob(SESSION_SNAPSHOT_CG, "cg_a"); + KafkaRoutineLoadJob jobB = newJob(SESSION_SNAPSHOT_CG, "cg_b"); + Map jobs = Maps.newHashMap(); + jobs.put(1L, jobA); + jobs.put(2L, jobB); + TestCloudRoutineLoadManager manager = new TestCloudRoutineLoadManager(jobs); + + List backendsForA = manager.getAvailableBackendIdsForTest(1L); + List backendsForB = manager.getAvailableBackendIdsForTest(2L); + + Assertions.assertEquals(Lists.newArrayList(beInA1.getId(), beInA2.getId()), backendsForA); + Assertions.assertEquals(Lists.newArrayList(beInB.getId()), backendsForB); + Assertions.assertTrue(backendsForA.stream().noneMatch(backendsForB::contains), + "jobs pinned to different compute groups must not share backends"); + } + + /** + * Isolation is driven by the declaration, not by the creating session: both jobs were created in + * the same session (same snapshot cluster) yet land on different backends. + */ + @Test + public void testDeclarationOverridesIdenticalCreatingSession() throws LoadException { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + + Backend beInA = createBackend(10001L); + Backend beInSession = createBackend(10009L); + Map> clusterToBackends = Maps.newHashMap(); + clusterToBackends.put("cg_a", Lists.newArrayList(beInA)); + clusterToBackends.put(SESSION_SNAPSHOT_CG, Lists.newArrayList(beInSession)); + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", + new ClusterAwareCloudSystemInfoService(clusterToBackends)); + + Map jobs = Maps.newHashMap(); + jobs.put(1L, newJob(SESSION_SNAPSHOT_CG, "cg_a")); + jobs.put(2L, newJob(SESSION_SNAPSHOT_CG, null)); + TestCloudRoutineLoadManager manager = new TestCloudRoutineLoadManager(jobs); + + Assertions.assertEquals(Lists.newArrayList(beInA.getId()), manager.getAvailableBackendIdsForTest(1L)); + Assertions.assertEquals(Lists.newArrayList(beInSession.getId()), manager.getAvailableBackendIdsForTest(2L)); + } + + // Changing the declaration takes effect on the next task allocation, no restart required. + @Test + public void testAlteringDeclarationMovesJobToAnotherComputeGroup() throws LoadException { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + + Backend beInA = createBackend(10001L); + Backend beInB = createBackend(10002L); + Map> clusterToBackends = Maps.newHashMap(); + clusterToBackends.put("cg_a", Lists.newArrayList(beInA)); + clusterToBackends.put("cg_b", Lists.newArrayList(beInB)); + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", + new ClusterAwareCloudSystemInfoService(clusterToBackends)); + + KafkaRoutineLoadJob job = newJob(SESSION_SNAPSHOT_CG, "cg_a"); + Map jobs = Maps.newHashMap(); + jobs.put(1L, job); + TestCloudRoutineLoadManager manager = new TestCloudRoutineLoadManager(jobs); + Assertions.assertEquals(Lists.newArrayList(beInA.getId()), manager.getAvailableBackendIdsForTest(1L)); + + // what ALTER ROUTINE LOAD ... PROPERTIES("compute_group" = "cg_b") ends up doing + Map jobProperties = Deencapsulation.getField(job, "jobProperties"); + jobProperties.put(RoutineLoadJob.COMPUTE_GROUP, "cg_b"); + + Assertions.assertEquals(Lists.newArrayList(beInB.getId()), manager.getAvailableBackendIdsForTest(1L)); + } + + /** + * Metadata load must not re-check the declared compute group. + * + *

{@link RoutineLoadJob#gsonPostProcess()} re-parses the stored CREATE statement on every FE + * metadata load - a restart, and every checkpoint - only to rebuild the RoutineLoadDesc, and it + * turns any failure into {@code JobState.CANCELLED}, which is final and cannot be undone by + * RESUME. A compute group can be dropped, renamed, or scaled to zero backends (which + * CloudSystemInfoService treats as dropped) while a job exists, so validating it there would + * kill the job permanently instead of letting the per task check pause it recoverably. + */ + @Test + public void testMetadataLoadDoesNotValidateDeclaredComputeGroup() throws UserException { + enterCloudMode(true, Lists.newArrayList("cg_live")); + + CreateRoutineLoadInfo info = newCreateInfo("cg_dropped"); + info.setReplay(true); + + info.checkJobProperties(); + + // Only the resource check is skipped: the declaration itself is still adopted, so the job + // keeps running in the group it was pinned to once the group comes back. + Assertions.assertEquals("cg_dropped", info.getComputeGroupName()); + // Nor does metadata load wait for the group to start: nothing is sent to it from here. + Assertions.assertTrue(cloudSystemInfo().autoStarted.isEmpty(), cloudSystemInfo().autoStarted.toString()); + } + + // Negative control for the test above: on the real CREATE path the same value must still be + // rejected, otherwise the skip would have disabled validation everywhere. + @Test + public void testCreateStillRejectsComputeGroupThatDoesNotExist() { + enterCloudMode(true, Lists.newArrayList("cg_live")); + + CreateRoutineLoadInfo info = newCreateInfo("cg_dropped"); + + UserException e = Assertions.assertThrows(UserException.class, info::checkJobProperties); + Assertions.assertTrue(e.getMessage().contains("Compute group 'cg_dropped' not found."), + e.getMessage()); + } + + /** + * Metadata load must not resolve the workload group either. + * + *

The workload group lookup resolves in the declared compute group + * ({@code ConnectContext#getComputeGroup} → {@code ComputeGroupMgr#getComputeGroupByName}), + * which throws once that group is gone, and {@link RoutineLoadJob#gsonPostProcess()} turns the + * throw into the final CANCELLED. The stored statement is the CREATE one: a job moved from + * cg_dropped to another group by ALTER still declares cg_dropped there, so dropping its old + * group used to cancel it on the next restart. + */ + @Test + public void testMetadataLoadDoesNotResolveWorkloadGroupInDeclaredComputeGroup() throws UserException { + enterCloudMode(true, Lists.newArrayList("cg_live")); + // resolve compute groups through the stub, where cg_dropped has no cluster id + Deencapsulation.setField(Env.getCurrentEnv(), "computeGroupMgr", new ComputeGroupMgr(cloudSystemInfo())); + + CreateRoutineLoadInfo info = newCreateInfo("cg_dropped", "wg_a"); + info.setReplay(true); + + info.checkJobProperties(); + + Assertions.assertEquals("cg_dropped", info.getComputeGroupName()); + Assertions.assertEquals("wg_a", info.getWorkloadGroupName()); + } + + /** + * Image load replays routine load jobs before any workload group exists. + * + *

{@code PersistMetaModules} reads {@code routineLoadJobs} before {@code workloadGroups}, and + * until {@code Env#loadWorkloadGroups} runs, the Env holds the empty WorkloadGroupMgr its + * constructor created. So the lookup above cannot find the group on image load even when no + * compute group is declared: a job created with a workload_group was cancelled by the first + * restart that loaded it from an image. This runs in non-cloud mode, where the replay context + * (ADMIN) looks the group up under the resource tag of every backend. + */ + @Test + public void testImageLoadAdoptsWorkloadGroupBeforeWorkloadGroupsAreLoaded() throws UserException { + List modules = PersistMetaModules.MODULE_NAMES; + Assertions.assertTrue(modules.indexOf("routineLoadJobs") < modules.indexOf("workloadGroups"), + modules.toString()); + mockAccessManager(true); + // the backends module is loaded before routineLoadJobs, the workload groups are not + SystemInfoService systemInfo = new SystemInfoService(); + systemInfo.addBackend(createBackend(10001L)); + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", systemInfo); + Deencapsulation.setField(Env.getCurrentEnv(), "computeGroupMgr", new ComputeGroupMgr(systemInfo)); + Deencapsulation.setField(Env.getCurrentEnv(), "workloadGroupMgr", new WorkloadGroupMgr()); + ConnectContext ctx = new ConnectContext(); + ctx.setCurrentUserIdentity(UserIdentity.ADMIN); + ctx.setThreadLocalInfo(); + + CreateRoutineLoadInfo info = newCreateInfo(null, "wg_a"); + info.setReplay(true); + info.checkJobProperties(); + Assertions.assertEquals("wg_a", info.getWorkloadGroupName()); + + // negative control: a CREATE naming a workload group that does not exist is still rejected + UserException e = Assertions.assertThrows(UserException.class, + newCreateInfo(null, "wg_a")::checkJobProperties); + Assertions.assertTrue(e.getMessage().contains("Can not find workload group wg_a"), e.getMessage()); + } + + /** + * CREATE wakes the declared group, as {@code USE @} does after the same checks. + * + *

With custom partitions, CREATE sends its Kafka metadata request through the declared + * group's backends while the statement runs. When that group is suspended and the session runs + * on another one, nothing else would start it, and the CREATE failed for lack of a backend. + */ + @Test + public void testCreateWakesDeclaredComputeGroup() throws UserException { + enterCloudMode(true, Lists.newArrayList("cg_live")); + + newCreateInfo("cg_live").checkJobProperties(); + + Assertions.assertEquals(Lists.newArrayList("cg_live"), cloudSystemInfo().autoStarted); + } + + // ALTER can move a job to a suspended group, and converts the new offsets in it right away. + @Test + public void testAlterWakesDeclaredComputeGroup() { + enterCloudMode(true, Lists.newArrayList("cg_live")); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.COMPUTE_GROUP, "cg_live"); + AlterRoutineLoadCommand command = new AlterRoutineLoadCommand(new LabelNameInfo("test_db", "test_job"), + jobProperties, Maps.newHashMap()); + + Deencapsulation.invoke(command, "checkJobProperties", ConnectContext.get()); + + Assertions.assertEquals("cg_live", command.getAnalyzedJobProperties().get(CreateRoutineLoadInfo.COMPUTE_GROUP)); + Assertions.assertEquals(Lists.newArrayList("cg_live"), cloudSystemInfo().autoStarted); + } + + /** + * {@code ALTER ROUTINE LOAD ... PROPERTIES("compute_group" = "cg_live") + * FROM KAFKA("kafka_partitions" = "0", "kafka_offsets" = "OFFSET_END")} converts the offsets + * before the new properties are installed, so the conversion has to go to cg_live already: the + * job's current group may be gone, which is why it is being moved. + */ + @Test + public void testCombinedAlterConvertsOffsetsInTheGroupItDeclares() { + enterCloudMode(true, Lists.newArrayList("cg_live")); + KafkaRoutineLoadJob job = newJob(SESSION_SNAPSHOT_CG, "cg_dropped"); + // left RUNNING, so modifyProperties stops at its PAUSED check right after the conversion + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.RUNNING); + + List conversionGroups = Lists.newArrayList(); + try (MockedStatic kafkaUtil = Mockito.mockStatic(KafkaUtil.class)) { + kafkaUtil.when(() -> KafkaUtil.getRealOffsets(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any(), Mockito.any())).thenAnswer(invocation -> { + conversionGroups.add(invocation.getArgument(4)); + return invocation.getArgument(3); + }); + + DdlException e = Assertions.assertThrows(DdlException.class, + () -> job.modifyProperties(newOffsetAlter("cg_live"))); + Assertions.assertTrue(e.getMessage().contains("Only supports modification of PAUSED jobs"), + e.getMessage()); + // an ALTER that declares no group keeps converting in the job's current one + Assertions.assertThrows(DdlException.class, () -> job.modifyProperties(newOffsetAlter(null))); + } + + Assertions.assertEquals(Lists.newArrayList("cg_live", "cg_dropped"), conversionGroups); + } + + // What AlterRoutineLoadCommand#validate leaves behind for "kafka_partitions" = "0", + // "kafka_offsets" = "OFFSET_END", optionally with a compute_group. + private AlterRoutineLoadCommand newOffsetAlter(String declaredComputeGroup) { + Map analyzedJobProperties = Maps.newHashMap(); + if (declaredComputeGroup != null) { + analyzedJobProperties.put(CreateRoutineLoadInfo.COMPUTE_GROUP, declaredComputeGroup); + } + KafkaDataSourceProperties dataSourceProperties = new KafkaDataSourceProperties(Maps.newHashMap()); + dataSourceProperties.setKafkaPartitionOffsets(Lists.newArrayList(Pair.of(0, KafkaProgress.OFFSET_END_VAL))); + AlterRoutineLoadCommand command = new AlterRoutineLoadCommand(new LabelNameInfo("test_db", "test_job"), + Maps.newHashMap(), Maps.newHashMap()); + Deencapsulation.setField(command, "analyzedJobProperties", analyzedJobProperties); + Deencapsulation.setField(command, "dataSourceProperties", dataSourceProperties); + return command; + } + + /** + * ALTER checks the job's workload group in the compute group it moves the job to, as CREATE does in the + * group it declares. + * + *

A workload group lives in the namespace of a compute group, and every task looks the job's one up in + * the group the job runs in ({@code KafkaTaskInfo#rePlan}). A job created with ("compute_group" = "cg_a", + * "workload_group" = "wg_a"), where only cg_a has wg_a, used to accept a move to cg_b. Every task then + * failed with "Can not find workload group wg_a in compute croup cg_b" and paused the job with the + * retryable CREATE_TASKS_ERR, which auto resume repeated for good. + */ + @Test + public void testAlterChecksWorkloadGroupInTheComputeGroupItDeclares() throws UserException { + enterCloudMode(true, Lists.newArrayList("cg_a", "cg_b")); + // only cg_a has wg_a, only cg_b has wg_b + List lookups = Lists.newArrayList(); + WorkloadGroupMgr workloadGroupMgr = Mockito.mock(WorkloadGroupMgr.class); + Mockito.when(workloadGroupMgr.getWorkloadGroup(Mockito.any(ConnectContext.class))).thenAnswer(invocation -> { + ConnectContext lookupCtx = invocation.getArgument(0); + String computeGroup = lookupCtx.getSessionVariable().getCloudCluster(); + String workloadGroup = lookupCtx.getSessionVariable().getWorkloadGroup(); + String lookup = computeGroup + "/" + workloadGroup; + lookups.add(lookupCtx.getCurrentUserIdentity().getQualifiedUser() + ":" + lookup); + if (!"cg_a/wg_a".equals(lookup) && !"cg_b/wg_b".equals(lookup)) { + throw new UserException("Can not find workload group " + workloadGroup + " in compute croup " + + computeGroup); + } + return Lists.newArrayList(Mockito.mock(WorkloadGroup.class)); + }); + Deencapsulation.setField(Env.getCurrentEnv(), "workloadGroupMgr", workloadGroupMgr); + + KafkaRoutineLoadJob job = newJob(SESSION_SNAPSHOT_CG, "cg_a"); + Map jobProperties = Deencapsulation.getField(job, "jobProperties"); + jobProperties.put(RoutineLoadJob.WORKLOAD_GROUP, "wg_a"); + // the tasks run as the owner, not as whoever runs the ALTER (the session here is admin) + Deencapsulation.setField(job, "userIdentity", UserIdentity.createAnalyzedUserIdentWithIp("owner", "%")); + // left RUNNING, so an ALTER that gets past the check stops at the PAUSED check right after it + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.RUNNING); + + // the move to cg_b keeps wg_a, which cg_b does not have + UserException e = Assertions.assertThrows(UserException.class, + () -> job.modifyProperties(newJobPropertiesAlter("cg_b", null))); + Assertions.assertTrue(e.getMessage().contains("Can not find workload group wg_a in compute croup cg_b"), + e.getMessage()); + Assertions.assertEquals("cg_a", job.getCloudCluster(), "a rejected ALTER must leave the job where it is"); + + // an ALTER that also names a workload group of cg_b passes the check + DdlException notPaused = Assertions.assertThrows(DdlException.class, + () -> job.modifyProperties(newJobPropertiesAlter("cg_b", "wg_b"))); + Assertions.assertTrue(notPaused.getMessage().contains("Only supports modification of PAUSED jobs"), + notPaused.getMessage()); + + // an ALTER that declares no compute group looks nothing up, as before + Assertions.assertThrows(DdlException.class, + () -> job.modifyProperties(newJobPropertiesAlter(null, "wg_unknown"))); + + Assertions.assertEquals(Lists.newArrayList("owner:cg_b/wg_a", "owner:cg_b/wg_b"), lookups); + } + + // What AlterRoutineLoadCommand#validate leaves behind for PROPERTIES("compute_group" = ..., + // "workload_group" = ...), with each property only when it is given. + private AlterRoutineLoadCommand newJobPropertiesAlter(String computeGroup, String workloadGroup) { + Map analyzedJobProperties = Maps.newHashMap(); + if (computeGroup != null) { + analyzedJobProperties.put(CreateRoutineLoadInfo.COMPUTE_GROUP, computeGroup); + } + if (workloadGroup != null) { + analyzedJobProperties.put(CreateRoutineLoadInfo.WORKLOAD_GROUP, workloadGroup); + } + AlterRoutineLoadCommand command = new AlterRoutineLoadCommand(new LabelNameInfo("test_db", "test_job"), + Maps.newHashMap(), Maps.newHashMap()); + Deencapsulation.setField(command, "analyzedJobProperties", analyzedJobProperties); + return command; + } + + /** + * The per task re-check has to run before the task takes any resource. + * + *

Backend allocation resolves the backends through the very compute group being checked, so + * a missing group makes it return an empty list and pause the job with a generic + * "no available BE found for job ... please check the BE status and user's cluster or tags". + * If the re-check ran after allocation it could never fire for a dropped group, and the + * operator would be sent to look at BE health for what is really a compute group that is gone. + * Running it first also means no transaction has been begun yet when it fails. + */ + @Test + public void testDroppedComputeGroupFailsTaskBeforeBackendAllocation() { + enterCloudMode(true, Lists.newArrayList("cg_live")); + + RecordingKafkaRoutineLoadJob job = new RecordingKafkaRoutineLoadJob(); + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.RUNNING); + Deencapsulation.setField(job, "userIdentity", UserIdentity.ADMIN); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(RoutineLoadJob.COMPUTE_GROUP, "cg_dropped"); + Deencapsulation.setField(job, "jobProperties", jobProperties); + + Map jobs = Maps.newHashMap(); + jobs.put(1L, job); + RoutineLoadTaskScheduler scheduler = + new RoutineLoadTaskScheduler(new TestCloudRoutineLoadManager(jobs)); + + ConcurrentMap partitionIdToOffset = Maps.newConcurrentMap(); + partitionIdToOffset.put(1, 100L); + KafkaTaskInfo taskInfo = new AlwaysReadyKafkaTaskInfo(new UUID(1, 1), 1L, partitionIdToOffset); + + try { + Deencapsulation.invoke(scheduler, "scheduleOneTask", taskInfo); + // Without the check in front, scheduling reaches allocateTaskToBe, which finds no + // backend for the dropped group, pauses the job with "no available BE found" and + // returns normally - so reaching this line is itself the regression. + Assertions.fail("scheduling a task of a job pinned to a dropped compute group must fail," + + " last pause reason: " + job.pauseMsg); + } catch (Exception expected) { + // scheduleOneTask pauses the job and rethrows + } + + Assertions.assertEquals(RoutineLoadJob.JobState.PAUSED, job.newState); + Assertions.assertNotNull(job.pauseMsg, "the job must be paused with a reason"); + Assertions.assertTrue(job.pauseMsg.contains("Compute group 'cg_dropped' not found."), job.pauseMsg); + Assertions.assertFalse(job.pauseMsg.contains("no available BE found"), + "the operator must not be sent to look at BE status for a dropped group"); + + // A group can come back by itself - one that is only scaled to zero backends leaves the + // cluster map and re-enters it when it scales up - so this pause has to stay retryable. + Assertions.assertNotEquals(InternalErrorCode.CANNOT_RESUME_ERR, job.pauseCode); + Assertions.assertTrue(ScheduleRule.isNeedAutoSchedule(job), + "a dropped group must still auto resume once it is back"); + } + + /** + * RESUME leaves the job in NEED_SCHEDULE, and the scheduler refreshes the topic's partitions before it creates + * any task. That metadata request goes to the backends of the declared group, so it runs the same check first. + */ + @Test + public void testDroppedComputeGroupPausesJobBeforeKafkaMetadataRequest() throws UserException { + enterCloudMode(true, Lists.newArrayList("cg_live")); + + RecordingKafkaRoutineLoadJob job = new RecordingKafkaRoutineLoadJob(); + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.NEED_SCHEDULE); + Deencapsulation.setField(job, "userIdentity", UserIdentity.ADMIN); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(RoutineLoadJob.COMPUTE_GROUP, "cg_dropped"); + Deencapsulation.setField(job, "jobProperties", jobProperties); + + try (MockedStatic kafkaUtil = Mockito.mockStatic(KafkaUtil.class)) { + // what the request reports for a group without backends + kafkaUtil.when(() -> KafkaUtil.getAllKafkaPartitions(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any())).thenThrow(new LoadException("failed to get info: no alive backends")); + + RoutineLoadJob routineLoadJob = job; + Assertions.assertFalse(routineLoadJob.refreshKafkaPartitions(false)); + + kafkaUtil.verifyNoInteractions(); + } + Assertions.assertEquals(RoutineLoadJob.JobState.PAUSED, job.newState); + Assertions.assertTrue(job.pauseMsg.contains("Compute group 'cg_dropped' not found."), job.pauseMsg); + Assertions.assertFalse(job.pauseMsg.contains("Kafka properties"), + "the operator must not be sent to check the Kafka properties for a dropped group"); + Assertions.assertTrue(ScheduleRule.isNeedAutoSchedule(job), + "a dropped group must still auto resume once it is back"); + } + + /** + * A revoked privilege is not transient, so the job must stop instead of flapping. + * + *

Pausing with the default INTERNAL_ERR would let {@link ScheduleRule} auto resume the job + * within at most MAX_BACK_OFF_TIME_SEC, whereupon the next task fails the same check and pauses + * it again - for as long as the grant is missing, at two edit log entries per cycle. + */ + @Test + public void testRevokedUsagePausesTaskWithoutAutoResume() { + // the group exists, so the check gets past existence and fails on the privilege + enterCloudMode(false, Lists.newArrayList("cg_pinned")); + + RecordingKafkaRoutineLoadJob job = new RecordingKafkaRoutineLoadJob(); + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.RUNNING); + Deencapsulation.setField(job, "userIdentity", UserIdentity.ADMIN); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(RoutineLoadJob.COMPUTE_GROUP, "cg_pinned"); + Deencapsulation.setField(job, "jobProperties", jobProperties); + + Map jobs = Maps.newHashMap(); + jobs.put(1L, job); + RoutineLoadTaskScheduler scheduler = + new RoutineLoadTaskScheduler(new TestCloudRoutineLoadManager(jobs)); + + ConcurrentMap partitionIdToOffset = Maps.newConcurrentMap(); + partitionIdToOffset.put(1, 100L); + KafkaTaskInfo taskInfo = new AlwaysReadyKafkaTaskInfo(new UUID(1, 1), 1L, partitionIdToOffset); + + try { + Deencapsulation.invoke(scheduler, "scheduleOneTask", taskInfo); + Assertions.fail("a task whose owner lost USAGE on the pinned group must fail"); + } catch (Exception expected) { + // scheduleOneTask pauses the job and rethrows + } + + Assertions.assertEquals(RoutineLoadJob.JobState.PAUSED, job.newState); + Assertions.assertNotNull(job.pauseMsg, "the job must be paused with a reason"); + Assertions.assertTrue(job.pauseMsg.contains("USAGE denied"), job.pauseMsg); + Assertions.assertEquals(InternalErrorCode.CANNOT_RESUME_ERR, job.pauseCode); + Assertions.assertFalse(ScheduleRule.isNeedAutoSchedule(job), + "a revoked privilege must not be auto resumed"); + } + + /** + * Auto resume leaves a revoked USAGE to the check before the first task, which records it. + * + *

Auto resume refreshes the partitions of a job that is still PAUSED, and a PAUSED job can not be paused + * again with another reason. A failing compute group check in that refresh therefore kept the job's old, + * retryable reason, and ScheduleRule went on resuming the job for good. The attempt now goes on to + * NEED_SCHEDULE, and the check before the first task pauses the job with CANNOT_RESUME_ERR. + */ + @Test + public void testAutoResumeLeavesRevokedUsageToTheTaskCheck() throws UserException { + // the group exists and its metadata request succeeds; only the owner's USAGE is gone + enterCloudMode(false, Lists.newArrayList("cg_pinned")); + + RecordingKafkaRoutineLoadJob job = new RecordingKafkaRoutineLoadJob(); + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.PAUSED); + // paused while the group was gone, which auto resume undoes once it is back + Deencapsulation.setField(job, "pauseReason", + new ErrorReason(InternalErrorCode.INTERNAL_ERR, "Compute group 'cg_pinned' not found.")); + Deencapsulation.setField(job, "userIdentity", UserIdentity.ADMIN); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(RoutineLoadJob.COMPUTE_GROUP, "cg_pinned"); + Deencapsulation.setField(job, "jobProperties", jobProperties); + Assertions.assertTrue(ScheduleRule.isNeedAutoSchedule(job), "a retryable pause is auto resumed"); + + try (MockedStatic kafkaUtil = Mockito.mockStatic(KafkaUtil.class)) { + kafkaUtil.when(() -> KafkaUtil.getAllKafkaPartitions(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any())).thenReturn(Lists.newArrayList(0)); + + RoutineLoadJob routineLoadJob = job; + Assertions.assertTrue(routineLoadJob.refreshKafkaPartitions(true), + "the attempt must go on to NEED_SCHEDULE instead of ending without a trace"); + kafkaUtil.verify(() -> KafkaUtil.getAllKafkaPartitions(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any())); + } + Assertions.assertNull(job.newState, "the refresh leaves the PAUSED job alone"); + + // update() moves the job to NEED_SCHEDULE, it is divided into tasks, and the first one is checked + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.RUNNING); + Map jobs = Maps.newHashMap(); + jobs.put(1L, job); + RoutineLoadTaskScheduler scheduler = + new RoutineLoadTaskScheduler(new TestCloudRoutineLoadManager(jobs)); + ConcurrentMap partitionIdToOffset = Maps.newConcurrentMap(); + partitionIdToOffset.put(0, 100L); + KafkaTaskInfo taskInfo = new AlwaysReadyKafkaTaskInfo(new UUID(1, 1), 1L, partitionIdToOffset); + // scheduleOneTask pauses the job and rethrows + Assertions.assertThrows(UserException.class, + () -> Deencapsulation.invoke(scheduler, "scheduleOneTask", taskInfo)); + + Assertions.assertEquals(RoutineLoadJob.JobState.PAUSED, job.newState); + Assertions.assertEquals(InternalErrorCode.CANNOT_RESUME_ERR, job.pauseCode); + Assertions.assertFalse(ScheduleRule.isNeedAutoSchedule(job), + "a revoked privilege must not be auto resumed"); + } + + /** + * update() refreshes the partitions before it takes the job lock, which a SQL STOP holds while it + * runs. A pause found by the refresh has to wait for that lock and look at the state again: a STOP + * that completed in between must stay STOPPED, as replay rejects STOPPED -> PAUSED. + */ + @Test + public void testRevokedUsagePauseInPartitionRefreshKeepsCompletedStop() throws Exception { + // the group exists and its metadata request would succeed; only the owner's USAGE is gone + enterCloudMode(false, Lists.newArrayList("cg_pinned")); + + RecordingKafkaRoutineLoadJob job = new RecordingKafkaRoutineLoadJob(); + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.NEED_SCHEDULE); + Deencapsulation.setField(job, "userIdentity", UserIdentity.ADMIN); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(RoutineLoadJob.COMPUTE_GROUP, "cg_pinned"); + Deencapsulation.setField(job, "jobProperties", jobProperties); + + Assertions.assertFalse(refreshWhileStopHoldsJobLock(job)); + + Assertions.assertNull(job.newState, "a completed STOP must not be turned back into PAUSED"); + Assertions.assertEquals(RoutineLoadJob.JobState.STOPPED, job.getState()); + } + + @Test + public void testKafkaMetadataPauseInPartitionRefreshKeepsCompletedStop() throws Exception { + RecordingKafkaRoutineLoadJob job = new RecordingKafkaRoutineLoadJob(); + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.NEED_SCHEDULE); + + try (MockedStatic kafkaUtil = Mockito.mockStatic(KafkaUtil.class)) { + kafkaUtil.when(() -> KafkaUtil.getAllKafkaPartitions(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any())).thenThrow(new LoadException("failed to get info")); + + Assertions.assertFalse(refreshWhileStopHoldsJobLock(job)); + } + + Assertions.assertNull(job.newState, "a completed STOP must not be turned back into PAUSED"); + Assertions.assertEquals(RoutineLoadJob.JobState.STOPPED, job.getState()); + } + + @Test + public void testKinesisShardPauseInRefreshKeepsCompletedStop() throws Exception { + RecordingKinesisRoutineLoadJob job = new RecordingKinesisRoutineLoadJob(); + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.NEED_SCHEDULE); + + try (MockedStatic kinesisUtil = Mockito.mockStatic(KinesisUtil.class)) { + kinesisUtil.when(() -> KinesisUtil.getAllKinesisShards(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any())).thenThrow(new LoadException("failed to list shards")); + + Assertions.assertFalse(refreshWhileStopHoldsJobLock(job)); + } + + Assertions.assertNull(job.newState, "a completed STOP must not be turned back into PAUSED"); + Assertions.assertEquals(RoutineLoadJob.JobState.STOPPED, job.getState()); + } + + /** + * Runs the partition refresh on this thread while another thread holds the job lock, as a SQL STOP + * does, and completes that STOP once the refresh waits for the lock. A refresh that changes the + * state without the lock never waits for it, and the STOP gives up after ten seconds. + */ + private static boolean refreshWhileStopHoldsJobLock(RoutineLoadJob job) throws Exception { + Thread refresher = Thread.currentThread(); + CountDownLatch stopHoldsLock = new CountDownLatch(1); + AtomicBoolean refreshWaitedForLock = new AtomicBoolean(); + Thread stop = new Thread(() -> { + job.writeLock(); + try { + stopHoldsLock.countDown(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!job.lock.hasQueuedThread(refresher) && System.nanoTime() < deadline) { + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + } + refreshWaitedForLock.set(job.lock.hasQueuedThread(refresher)); + // what executeStop() leaves behind + job.state = RoutineLoadJob.JobState.STOPPED; + } finally { + job.writeUnlock(); + } + }); + stop.start(); + stopHoldsLock.await(); + boolean refreshed = job.refreshKafkaPartitions(false); + stop.join(); + Assertions.assertTrue(refreshWaitedForLock.get(), "the pause must wait for the job lock"); + return refreshed; + } + + private CreateRoutineLoadInfo newCreateInfo(String declaredComputeGroup) { + return newCreateInfo(declaredComputeGroup, null); + } + + private CreateRoutineLoadInfo newCreateInfo(String declaredComputeGroup, String workloadGroup) { + Map jobProperties = Maps.newHashMap(); + if (declaredComputeGroup != null) { + jobProperties.put(CreateRoutineLoadInfo.COMPUTE_GROUP, declaredComputeGroup); + } + if (workloadGroup != null) { + jobProperties.put(CreateRoutineLoadInfo.WORKLOAD_GROUP, workloadGroup); + } + Map dataSourceProperties = Maps.newHashMap(); + dataSourceProperties.put("kafka_broker_list", "127.0.0.1:9092"); + dataSourceProperties.put("kafka_topic", "test_topic"); + Map loadPropertyMap = Maps.newHashMap(); + return new CreateRoutineLoadInfo(new LabelNameInfo("test_db", "test_job"), "test_tbl", + loadPropertyMap, jobProperties, "kafka", dataSourceProperties, + LoadTask.MergeType.APPEND, ""); + } + + private Backend createBackend(long id) { + Backend backend = new Backend(id, "127.0.0." + id, 9050); + backend.setAlive(true); + return backend; + } + + /** + * KafkaTaskInfo asks the real RoutineLoadManager, and then Kafka, whether there is more data to + * consume. Neither exists here, so answer yes and let scheduling proceed to the part under + * test: without the compute group check in front, it must reach backend allocation. + */ + private static class AlwaysReadyKafkaTaskInfo extends KafkaTaskInfo { + private AlwaysReadyKafkaTaskInfo(UUID id, long jobId, Map partitionIdToOffset) { + super(id, jobId, 20000, partitionIdToOffset, false, -1, false); + } + + @Override + protected boolean hasMoreDataToConsume() { + return true; + } + } + + /** + * Captures the state transition instead of writing an edit log, and mirrors it onto the real + * fields so that ScheduleRule can be asked what it would do with the resulting pause. + */ + private static class RecordingKafkaRoutineLoadJob extends KafkaRoutineLoadJob { + private JobState newState; + private String pauseMsg; + private InternalErrorCode pauseCode; + + @Override + public void updateState(JobState jobState, ErrorReason reason, boolean isReplay) { + this.newState = jobState; + this.pauseMsg = reason == null ? null : reason.getMsg(); + this.pauseCode = reason == null ? null : reason.getCode(); + this.state = jobState; + this.pauseReason = reason; + } + + // the partition refresh changes the state under a lock it takes itself + @Override + protected void unprotectUpdateState(JobState jobState, ErrorReason reason, boolean isReplay) { + updateState(jobState, reason, isReplay); + } + } + + private static class RecordingKinesisRoutineLoadJob extends KinesisRoutineLoadJob { + private JobState newState; + + private RecordingKinesisRoutineLoadJob() { + super(1L, "kinesis_routine_load_job", 1L, 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); + } + + @Override + protected void unprotectUpdateState(JobState jobState, ErrorReason reason, boolean isReplay) { + this.newState = jobState; + this.state = jobState; + } + } + + private static class ClusterAwareCloudSystemInfoService extends CloudSystemInfoService { + private final Map> clusterToBackends; + // every group asked to start, in call order; the real call resumes a suspended group + private final List autoStarted = Lists.newArrayList(); + + private ClusterAwareCloudSystemInfoService(Map> clusterToBackends) { + this.clusterToBackends = clusterToBackends; + } + + @Override + public String waitForAutoStart(String clusterName) { + autoStarted.add(clusterName); + return clusterName; + } + + @Override + public List getBackendsByClusterName(final String clusterName) { + return clusterToBackends.getOrDefault(clusterName, Lists.newArrayList()); + } + + @Override + public List getCloudClusterNames() { + return clusterToBackends.keySet().stream().collect(Collectors.toList()); + } + } + + private static class TestCloudRoutineLoadManager extends CloudRoutineLoadManager { + private final Map jobs; + + private TestCloudRoutineLoadManager(Map jobs) { + this.jobs = jobs; + } + + @Override + public RoutineLoadJob getJob(long jobId) { + return jobs.get(jobId); + } + + // the real one looks the task up in idToRoutineLoadJob, which this stub never fills + @Override + public boolean checkTaskInJob(RoutineLoadTaskInfo task) { + return jobs.containsKey(task.getJobId()); + } + + private List getAvailableBackendIdsForTest(long jobId) throws LoadException { + return super.getAvailableBackendIds(jobId); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java index e1f25b14dd0dbe..87a9d33c27f830 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java @@ -33,6 +33,7 @@ import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; import org.apache.doris.load.routineload.kafka.KafkaTaskInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.EditLog; import org.apache.doris.persist.RoutineLoadOperation; import org.apache.doris.qe.OriginStatement; @@ -379,6 +380,50 @@ public void testGetShowCreateInfo() throws UserException { Assertions.assertEquals(expect, showCreateInfo); } + // The statement carries the compute group the job declares, as the last ALTER left it, so a job + // recreated from it stays pinned instead of running in the recreating session's group. + @Test + public void testGetShowCreateInfoCarriesDeclaredComputeGroup() throws UserException { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(111L, "test_load", 1, + 11, "localhost:9092", "test_topic", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "maxErrorNum", 10); + Deencapsulation.setField(routineLoadJob, "maxBatchRows", 10); + // what CREATE ROUTINE LOAD ... PROPERTIES("compute_group" = "cg_a") stores + Map jobProperties = Deencapsulation.getField(routineLoadJob, "jobProperties"); + jobProperties.put(RoutineLoadJob.COMPUTE_GROUP, "cg_a"); + // ALTER ROUTINE LOAD ... PROPERTIES("compute_group" = "cg_b"); modifyProperties and its replay + // install the new properties through the same code, the replay only writes no edit log + Map alteredProperties = Maps.newHashMap(); + alteredProperties.put(CreateRoutineLoadInfo.COMPUTE_GROUP, "cg_b"); + routineLoadJob.replayModifyProperties(new AlterRoutineLoadJobOperationLog(111L, alteredProperties, null)); + + String expect = "CREATE ROUTINE LOAD test_load ON 11\n" + + "WITH APPEND\n" + + "PROPERTIES\n" + + "(\n" + + "\"desired_concurrent_number\" = \"0\",\n" + + "\"max_error_number\" = \"10\",\n" + + "\"max_filter_ratio\" = \"1.0\",\n" + + "\"max_batch_interval\" = \"60\",\n" + + "\"max_batch_rows\" = \"10\",\n" + + "\"max_batch_size\" = \"1073741824\",\n" + + "\"format\" = \"csv\",\n" + + "\"strip_outer_array\" = \"false\",\n" + + "\"num_as_string\" = \"false\",\n" + + "\"fuzzy_parse\" = \"false\",\n" + + "\"strict_mode\" = \"false\",\n" + + "\"timezone\" = \"Asia/Shanghai\",\n" + + "\"compute_group\" = \"cg_b\",\n" + + "\"exec_mem_limit\" = \"2147483648\"\n" + + ")\n" + + "FROM KAFKA\n" + + "(\n" + + "\"kafka_broker_list\" = \"localhost:9092\",\n" + + "\"kafka_topic\" = \"test_topic\"\n" + + ");"; + Assertions.assertEquals(expect, routineLoadJob.getShowCreateInfo()); + } + @Test public void testParseUniqueKeyUpdateMode() { // Test valid mode strings diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVComputeGroupTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVComputeGroupTest.java new file mode 100644 index 00000000000000..69c7639dc29fdd --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVComputeGroupTest.java @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.info.TableNameInfo; +import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Config; +import org.apache.doris.common.UserException; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.job.extensions.mtmv.MTMVTask; +import org.apache.doris.job.extensions.mtmv.MTMVTask.MTMVTaskTriggerMode; +import org.apache.doris.job.extensions.mtmv.MTMVTaskContext; +import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo.RefreshMode; +import org.apache.doris.persist.AlterMTMV; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.SystemInfoService; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.Optional; + +/** + * The transitional {@code compute_group} declaration on async materialized views. + * + *

A declared compute group pins every refresh - automatic and manual alike - to that group. + * Only when nothing is declared does a manual REFRESH keep borrowing the triggering session's group, + * which is the behaviour every existing MV keeps. + */ +public class MTMVComputeGroupTest { + + private static final String DECLARED_CG = "cg_declared"; + private static final String SESSION_CG = "cg_from_session"; + + private String originalDeployMode; + private String originalCloudUniqueId; + private SystemInfoService originalSystemInfo; + + @BeforeEach + public void setUp() { + originalDeployMode = Config.deploy_mode; + originalCloudUniqueId = Config.cloud_unique_id; + originalSystemInfo = Env.getCurrentSystemInfo(); + } + + @AfterEach + public void tearDown() { + Config.deploy_mode = originalDeployMode; + Config.cloud_unique_id = originalCloudUniqueId; + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", originalSystemInfo); + ConnectContext.remove(); + } + + private MTMV newMTMV(String declaredComputeGroup) { + MTMV mtmv = new MTMV(); + Map mvProperties = Maps.newHashMap(); + if (declaredComputeGroup != null) { + mvProperties.put(PropertyAnalyzer.PROPERTIES_COMPUTE_GROUP, declaredComputeGroup); + } + mtmv.setMvProperties(mvProperties); + return mtmv; + } + + private MTMVTask newTask(MTMV mtmv, MTMVTaskTriggerMode triggerMode, String taskContextComputeGroup) { + MTMVTaskContext taskContext = MTMVTaskContext.of( + triggerMode, Lists.newArrayList(), RefreshMode.AUTO, true, taskContextComputeGroup); + return new MTMVTask(mtmv, null, taskContext); + } + + private String resolveComputeGroup(MTMVTask task) { + ConnectContext ctx = new ConnectContext(); + Deencapsulation.invoke(task, "applyComputeGroup", ctx); + return ctx.getSessionVariable().getCloudCluster(); + } + + @Test + public void testGetComputeGroupFromProperty() { + Assertions.assertEquals(DECLARED_CG, newMTMV(DECLARED_CG).getComputeGroup().orElse(null)); + Assertions.assertFalse(newMTMV(null).getComputeGroup().isPresent()); + Assertions.assertFalse(newMTMV("").getComputeGroup().isPresent()); + } + + @Test + public void testAlterComputeGroupProperty() { + MTMV mtmv = newMTMV(DECLARED_CG); + Map changed = Maps.newHashMap(); + changed.put(PropertyAnalyzer.PROPERTIES_COMPUTE_GROUP, "cg_after_alter"); + AlterMTMV alterMTMV = new AlterMTMV(new TableNameInfo("db", "mv"), MTMVAlterOpType.ALTER_PROPERTY); + alterMTMV.setMvProperties(changed); + // A replay applies the property change without writing an edit log. + mtmv.alterMvProperties(alterMTMV, true); + Assertions.assertEquals("cg_after_alter", mtmv.getComputeGroup().orElse(null)); + } + + // Automatic refresh has no compute group on the task context; the declaration is what pins it. + @Test + public void testAutoRefreshUsesDeclaredComputeGroup() { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + MTMVTask task = newTask(newMTMV(DECLARED_CG), MTMVTaskTriggerMode.SYSTEM, null); + Assertions.assertEquals(DECLARED_CG, resolveComputeGroup(task)); + } + + // The one deliberate behaviour change: a declared compute group also wins for a manual REFRESH, + // so the same MV never refreshes in two different places depending on who triggered it. + @Test + public void testManualRefreshPrefersDeclaredOverSessionComputeGroup() { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + MTMVTask task = newTask(newMTMV(DECLARED_CG), MTMVTaskTriggerMode.MANUAL, SESSION_CG); + Assertions.assertEquals(DECLARED_CG, resolveComputeGroup(task)); + } + + // Existing MVs declare nothing, so a manual REFRESH must keep borrowing the session's group. + @Test + public void testManualRefreshFallsBackToSessionComputeGroup() { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + MTMVTask task = newTask(newMTMV(null), MTMVTaskTriggerMode.MANUAL, SESSION_CG); + Assertions.assertEquals(SESSION_CG, resolveComputeGroup(task)); + } + + // Nothing declared and nothing on the task context leaves the context untouched, which keeps the + // existing implicit resolution (admin's default compute group, or one picked by policy). + @Test + public void testAutoRefreshWithoutDeclarationLeavesContextUntouched() { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + MTMVTask task = newTask(newMTMV(null), MTMVTaskTriggerMode.SYSTEM, null); + // untouched, i.e. left at the session variable's default so the implicit resolution applies + Assertions.assertTrue(Strings.isNullOrEmpty(resolveComputeGroup(task))); + } + + // Non-cloud is out of scope; the declaration can not be created there, and even if metadata + // somehow carried one it must not touch the context. + @Test + public void testNoopInNonCloudMode() { + Config.deploy_mode = ""; + Config.cloud_unique_id = ""; + MTMVTask task = newTask(newMTMV(DECLARED_CG), MTMVTaskTriggerMode.MANUAL, SESSION_CG); + Assertions.assertTrue(Strings.isNullOrEmpty(resolveComputeGroup(task))); + } + + /** + * Isolation: two MVs declaring different compute groups resolve to different clusters, which is + * what makes their refreshes pick disjoint backend sets downstream. + */ + @Test + public void testMvsWithDifferentComputeGroupsResolveIndependently() { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + MTMVTask taskA = newTask(newMTMV("cg_a"), MTMVTaskTriggerMode.SYSTEM, null); + MTMVTask taskB = newTask(newMTMV("cg_b"), MTMVTaskTriggerMode.SYSTEM, null); + + String resolvedA = resolveComputeGroup(taskA); + String resolvedB = resolveComputeGroup(taskB); + Assertions.assertEquals("cg_a", resolvedA); + Assertions.assertEquals("cg_b", resolvedB); + Assertions.assertNotEquals(resolvedA, resolvedB); + } + + // Both refresh paths, IVM incremental and partition / complete, set their context up through + // setupComputeGroup(), which must apply the declaration and record the group the refresh runs in. + @Test + public void testRefreshContextSetupAppliesAndRecordsDeclaredComputeGroup() { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + MTMVTask task = newTask(newMTMV(DECLARED_CG), MTMVTaskTriggerMode.MANUAL, SESSION_CG); + ConnectContext ctx = new ConnectContext(); + Deencapsulation.invoke(task, "setupComputeGroup", ctx); + Assertions.assertEquals(DECLARED_CG, ctx.getSessionVariable().getCloudCluster()); + Assertions.assertEquals(DECLARED_CG, Deencapsulation.getField(task, "computeGroup")); + } + + // A refresh re-checks the group the MV declares, which can be dropped while the MV exists. An MV + // that declares none is not checked: it never chose the group it borrows from the session. + @Test + public void testRefreshRechecksOnlyTheDeclaredComputeGroup() throws Exception { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + CloudSystemInfoService systemInfo = Mockito.mock(CloudSystemInfoService.class); + Mockito.when(systemInfo.getCloudClusterNames()).thenReturn(Lists.newArrayList(SESSION_CG)); + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", systemInfo); + // both refresh paths set their context up, and re-check it, through setupComputeGroup() + Method check = MTMVTask.class.getDeclaredMethod("setupComputeGroup", ConnectContext.class); + check.setAccessible(true); + ConnectContext ctx = new ConnectContext(); + ctx.setCurrentUserIdentity(UserIdentity.ADMIN); + + MTMVTask declaredButDropped = newTask(newMTMV(DECLARED_CG), MTMVTaskTriggerMode.SYSTEM, null); + InvocationTargetException e = Assertions.assertThrows(InvocationTargetException.class, + () -> check.invoke(declaredButDropped, ctx)); + Assertions.assertInstanceOf(UserException.class, e.getCause()); + Assertions.assertTrue(e.getCause().getMessage().contains("Compute group '" + DECLARED_CG + "' not found."), + e.getCause().getMessage()); + + check.invoke(newTask(newMTMV(null), MTMVTaskTriggerMode.MANUAL, "cg_dropped_from_session"), ctx); + } + + /** + * A refresh context is re-checked against the group it is bound to. + * + *

The declaration is read once per context. The check used to read it a second time, so an ALTER + * MATERIALIZED VIEW landing in between had it pass for the new group while the refresh ran in the old one. + * When the old group had been dropped, which is what such an ALTER is often for, the refresh then failed + * later on a generic "no backend" error instead of the "not found" this check exists to give. + */ + @Test + public void testRefreshContextChecksTheGroupItBinds() { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + // cg_a has been dropped, cg_b exists + CloudSystemInfoService systemInfo = Mockito.mock(CloudSystemInfoService.class); + Mockito.when(systemInfo.getCloudClusterNames()).thenReturn(Lists.newArrayList("cg_b")); + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", systemInfo); + // the first read sees cg_a, any later one the ALTER to cg_b that lands right after it + MTMV mtmv = Mockito.spy(newMTMV("cg_a")); + Mockito.doReturn(Optional.of("cg_a"), Optional.of("cg_b")).when(mtmv).getComputeGroup(); + MTMVTask task = newTask(mtmv, MTMVTaskTriggerMode.SYSTEM, null); + ConnectContext ctx = new ConnectContext(); + ctx.setCurrentUserIdentity(UserIdentity.ADMIN); + + UserException e = Assertions.assertThrows(UserException.class, + () -> Deencapsulation.invoke(task, "setupComputeGroup", ctx)); + Assertions.assertTrue(e.getMessage().contains("Compute group 'cg_a' not found."), e.getMessage()); + Assertions.assertEquals("cg_a", ctx.getSessionVariable().getCloudCluster()); + Mockito.verify(mtmv, Mockito.times(1)).getComputeGroup(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommandTest.java index 81becc8b5a9c66..e56e67909d324d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommandTest.java @@ -17,10 +17,19 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.analysis.ResourceTypeEnum; import org.apache.doris.analysis.StatementBase; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.info.TableNameInfo; +import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Config; +import org.apache.doris.common.UserException; +import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.mtmv.ivm.IvmIncrRefreshManager; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.parser.NereidsParser; @@ -29,13 +38,19 @@ import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.SystemInfoService; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import java.lang.reflect.Field; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -44,8 +59,33 @@ */ public class RefreshMTMVCommandTest { + private static final String DECLARED_CG = "cg_declared"; + private final NereidsParser parser = new NereidsParser(); + private String originalDeployMode; + private String originalCloudUniqueId; + private SystemInfoService originalSystemInfo; + private AccessControllerManager originalAccessManager; + + @BeforeEach + public void setUp() { + originalDeployMode = Config.deploy_mode; + originalCloudUniqueId = Config.cloud_unique_id; + originalSystemInfo = Env.getCurrentSystemInfo(); + originalAccessManager = Env.getCurrentEnv().getAccessManager(); + } + + @AfterEach + public void tearDown() { + Config.deploy_mode = originalDeployMode; + Config.cloud_unique_id = originalCloudUniqueId; + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", originalSystemInfo); + Deencapsulation.setField(Env.getCurrentEnv(), "accessManager", originalAccessManager); + // the dry run and EXPLAIN contexts install themselves as the thread-local context + ConnectContext.remove(); + } + private RefreshMTMVInfo extractRefreshInfo(String sql) throws Exception { StatementBase statementBase = parser.parseSQL(sql).get(0); Assertions.assertTrue(statementBase instanceof LogicalPlanAdapter, @@ -211,6 +251,83 @@ public InsertIntoTableCommand buildInsertCommand(MTMV mtmv) { Assertions.assertEquals(excluded, stmtCtx.getExcludedTriggerTables()); } + // ---- the compute group the dry run and EXPLAIN REFRESH contexts run in ---- + + // Cloud mode with these compute groups, on all of which admin, the identity of both contexts, + // holds USAGE. + private void enterCloudMode(List existingComputeGroups) { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + CloudSystemInfoService systemInfo = Mockito.mock(CloudSystemInfoService.class); + Mockito.when(systemInfo.getCloudClusterNames()).thenReturn(existingComputeGroups); + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", systemInfo); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.when(accessManager.checkCloudPriv(Mockito.any(UserIdentity.class), Mockito.anyString(), + Mockito.any(PrivPredicate.class), Mockito.any(ResourceTypeEnum.class))).thenReturn(true); + Deencapsulation.setField(Env.getCurrentEnv(), "accessManager", accessManager); + } + + private MTMV newMtmvDeclaring(String declaredComputeGroup) { + MTMV mtmv = Mockito.mock(MTMV.class); + Mockito.when(mtmv.getComputeGroup()).thenReturn(Optional.ofNullable(declaredComputeGroup)); + return mtmv; + } + + private TestRefreshMTMVCommand newIncrementalCommand(boolean dryRun) throws Exception { + return new TestRefreshMTMVCommand(extractRefreshInfo("REFRESH MATERIALIZED VIEW db1.mv1 INCREMENTAL"), dryRun); + } + + // The dry run plans and executes its delta query, and NereidsCoordinator looks the MV's workload + // group up, in the compute group of this context. It must be the group the MV declares, where + // every refresh of the MV runs, not the one admin resolves to. + @Test + public void testDryRunContextBindsDeclaredComputeGroup() throws Exception { + enterCloudMode(Lists.newArrayList("cg_of_admin", DECLARED_CG)); + + ConnectContext internalCtx = newIncrementalCommand(true).createDryRunConnectContext( + newMtmvDeclaring(DECLARED_CG)); + + Assertions.assertEquals(DECLARED_CG, internalCtx.getSessionVariable().getCloudCluster()); + } + + // The declared group can be dropped while the MV exists. Like a real refresh, the dry run then + // fails with that reason instead of running the delta in another group. + @Test + public void testDryRunRejectsDroppedDeclaredComputeGroup() throws Exception { + enterCloudMode(Lists.newArrayList("cg_of_admin")); + TestRefreshMTMVCommand command = newIncrementalCommand(true); + + UserException e = Assertions.assertThrows(UserException.class, + () -> command.createDryRunConnectContext(newMtmvDeclaring(DECLARED_CG))); + + Assertions.assertTrue(e.getMessage().contains("Compute group '" + DECLARED_CG + "' not found."), + e.getMessage()); + } + + // EXPLAIN REFRESH only plans, but planning assigns backends, so it plans in the declared group too. + @Test + public void testExplainContextBindsDeclaredComputeGroup() throws Exception { + enterCloudMode(Lists.newArrayList("cg_of_admin", DECLARED_CG)); + + ConnectContext explainCtx = newIncrementalCommand(false).createExplainConnectContext( + newMtmvDeclaring(DECLARED_CG)); + + Assertions.assertEquals(DECLARED_CG, explainCtx.getSessionVariable().getCloudCluster()); + } + + // An MV that declares nothing keeps the existing resolution: neither context names a group, and + // the dry run has nothing to re-check, even with no compute group left at all. + @Test + public void testContextsWithoutDeclarationKeepImplicitComputeGroup() throws Exception { + enterCloudMode(Lists.newArrayList()); + MTMV mtmv = newMtmvDeclaring(null); + + Assertions.assertTrue(Strings.isNullOrEmpty(newIncrementalCommand(true) + .createDryRunConnectContext(mtmv).getSessionVariable().getCloudCluster())); + Assertions.assertTrue(Strings.isNullOrEmpty(newIncrementalCommand(false) + .createExplainConnectContext(mtmv).getSessionVariable().getCloudCluster())); + } + private static class TestRefreshMTMVCommand extends RefreshMTMVCommand { private final IvmIncrRefreshManager manager; diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java index 759d56a285b62a..06fa707fcc243e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java @@ -336,7 +336,7 @@ public void testThreadInfoUsesSessionCloudCluster() { ConnectContext ctx = new ConnectContext(); ctx.cloudCluster = "test"; - ctx.setCloudCluster("test1"); + ctx.getSessionVariable().setCloudCluster("test1"); List row = ctx.toThreadInfo(false).toRow(-1, 0, Optional.empty()); Assertions.assertEquals("test1", row.get(14)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/resource/computegroup/ComputeGroupBindingUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/resource/computegroup/ComputeGroupBindingUtilTest.java new file mode 100644 index 00000000000000..df9e9c2a06722d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/resource/computegroup/ComputeGroupBindingUtilTest.java @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.resource.computegroup; + +import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Config; +import org.apache.doris.common.InternalErrorCode; +import org.apache.doris.common.UserException; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.SystemInfoService; + +import com.google.common.collect.Lists; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.List; + +/** + * Validation of the transitional {@code compute_group} declaration. + */ +public class ComputeGroupBindingUtilTest { + + private static final String CG_OK = "cg_ok"; + + private String originalDeployMode; + private String originalCloudUniqueId; + private SystemInfoService originalSystemInfo; + private SystemInfoService originalCgMgrSystemInfo; + private AccessControllerManager originalAccessManager; + private ConnectContext ctx; + + @BeforeEach + public void setUp() { + originalDeployMode = Config.deploy_mode; + originalCloudUniqueId = Config.cloud_unique_id; + originalSystemInfo = Env.getCurrentSystemInfo(); + originalCgMgrSystemInfo = Deencapsulation.getField( + Env.getCurrentEnv().getComputeGroupMgr(), "systemInfoService"); + originalAccessManager = Env.getCurrentEnv().getAccessManager(); + ctx = new ConnectContext(); + ctx.setCurrentUserIdentity(UserIdentity.ADMIN); + } + + @AfterEach + public void tearDown() { + Config.deploy_mode = originalDeployMode; + Config.cloud_unique_id = originalCloudUniqueId; + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", originalSystemInfo); + Deencapsulation.setField(Env.getCurrentEnv().getComputeGroupMgr(), "systemInfoService", + originalCgMgrSystemInfo); + Deencapsulation.setField(Env.getCurrentEnv(), "accessManager", originalAccessManager); + ConnectContext.remove(); + } + + private void enterCloudMode(boolean hasPriv, List existingClusters) { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.when(accessManager.checkCloudPriv(Mockito.any(UserIdentity.class), Mockito.anyString(), + Mockito.any(PrivPredicate.class), Mockito.any(ResourceTypeEnum.class))).thenReturn(hasPriv); + Deencapsulation.setField(Env.getCurrentEnv(), "accessManager", accessManager); + TestCloudSystemInfoService svc = new TestCloudSystemInfoService(existingClusters); + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", svc); + // ComputeGroupMgr captured its own reference at construction, so replacing only Env's would + // leave getComputeGroupByName() casting the real non-cloud service. + Deencapsulation.setField(Env.getCurrentEnv().getComputeGroupMgr(), "systemInfoService", svc); + } + + // An empty declaration means "not declared" and must never fail, not even in non-cloud mode, + // otherwise every existing job would break. + @Test + public void testEmptyDeclarationIsNoOp() throws UserException { + Config.deploy_mode = ""; + Config.cloud_unique_id = ""; + ComputeGroupBindingUtil.validateDeclaredComputeGroup(ctx, null); + ComputeGroupBindingUtil.validateDeclaredComputeGroup(ctx, ""); + } + + // Non-cloud is out of scope for this transitional change: declaring a compute group must be + // rejected so that no non-cloud metadata ever carries the key. + @Test + public void testRejectedInNonCloudMode() { + Config.deploy_mode = ""; + Config.cloud_unique_id = ""; + UserException e = Assertions.assertThrows(UserException.class, + () -> ComputeGroupBindingUtil.validateDeclaredComputeGroup(ctx, CG_OK)); + Assertions.assertTrue(e.getMessage().contains("only supported in cloud mode"), e.getMessage()); + } + + // DEFAULT is reserved by the final binding design ("follow the owner's default group"). + // Pinning a group literally named DEFAULT would silently change behaviour after upgrading. + @Test + public void testRejectReservedDefaultValue() { + enterCloudMode(true, Lists.newArrayList("DEFAULT", "default", CG_OK)); + for (String reserved : new String[] {"DEFAULT", "default", "Default"}) { + UserException e = Assertions.assertThrows(UserException.class, + () -> ComputeGroupBindingUtil.validateDeclaredComputeGroup(ctx, reserved)); + Assertions.assertTrue(e.getMessage().contains("reserved value"), e.getMessage()); + } + } + + @Test + public void testRejectWhenNoUsagePrivilege() { + enterCloudMode(false, Lists.newArrayList(CG_OK)); + UserException e = Assertions.assertThrows(UserException.class, + () -> ComputeGroupBindingUtil.validateDeclaredComputeGroup(ctx, CG_OK)); + Assertions.assertTrue(e.getMessage().contains("USAGE denied"), e.getMessage()); + } + + @Test + public void testRejectWhenComputeGroupDoesNotExist() { + enterCloudMode(true, Lists.newArrayList("some_other_cg")); + UserException e = Assertions.assertThrows(UserException.class, + () -> ComputeGroupBindingUtil.validateDeclaredComputeGroup(ctx, CG_OK)); + Assertions.assertTrue(e.getMessage().contains("not found"), e.getMessage()); + } + + @Test + public void testAcceptValidDeclaration() throws UserException { + enterCloudMode(true, Lists.newArrayList(CG_OK, "cg_other")); + ComputeGroupBindingUtil.validateDeclaredComputeGroup(ctx, CG_OK); + } + + // The privilege check must run before the existence check, so that probing for the existence of + // a compute group the user has no access to is not possible. + @Test + public void testPrivilegeIsCheckedBeforeExistence() { + enterCloudMode(false, Lists.newArrayList("some_other_cg")); + UserException e = Assertions.assertThrows(UserException.class, + () -> ComputeGroupBindingUtil.validateDeclaredComputeGroup(ctx, CG_OK)); + Assertions.assertTrue(e.getMessage().contains("USAGE denied"), e.getMessage()); + Assertions.assertFalse(e.getMessage().contains("not found"), e.getMessage()); + } + + // ---- checkComputeGroupBeforeTask: re-checked before every task, against the job's owner ---- + + // Jobs created before the owner was persisted have nothing to check against. + @Test + public void testRuntimeCheckSkippedWhenOwnerUnknown() throws UserException { + enterCloudMode(false, Lists.newArrayList()); + ComputeGroupBindingUtil.checkComputeGroupBeforeTask(null, "cg_gone"); + } + + // Nothing is bound, so there is nothing to re-check. + @Test + public void testRuntimeCheckSkippedWhenNothingBound() throws UserException { + enterCloudMode(false, Lists.newArrayList()); + ComputeGroupBindingUtil.checkComputeGroupBeforeTask(UserIdentity.ADMIN, null); + } + + // The compute group was dropped while the job kept running. + // + // This runs without a thread-local ConnectContext on purpose: the callers are background + // threads (the routine load scheduler, the MV task runner) that have none, so the check must + // not reach any code path that needs one. + @Test + public void testRuntimeCheckFailsWhenComputeGroupDropped() { + enterCloudMode(true, Lists.newArrayList()); + ConnectContext.remove(); + UserException e = Assertions.assertThrows(UserException.class, + () -> ComputeGroupBindingUtil.checkComputeGroupBeforeTask(UserIdentity.ADMIN, "cg_gone")); + Assertions.assertTrue(e.getMessage().contains("not found"), e.getMessage()); + // A missing group can come back on its own - one that is only scaled to zero backends + // leaves the cluster map and re-enters it when it scales up - so the pause this produces + // has to stay retryable for RoutineLoadTaskScheduler / ScheduleRule. + Assertions.assertNotEquals(InternalErrorCode.CANNOT_RESUME_ERR, e.getErrorCode()); + } + + // The owner's USAGE on the compute group was revoked while the job kept running. This is the + // case creation-time validation alone cannot catch. + @Test + public void testRuntimeCheckFailsWhenComputeGroupUsageRevoked() { + enterCloudMode(false, Lists.newArrayList(CG_OK)); + UserException e = Assertions.assertThrows(UserException.class, + () -> ComputeGroupBindingUtil.checkComputeGroupBeforeTask(UserIdentity.ADMIN, CG_OK)); + Assertions.assertTrue(e.getMessage().contains("USAGE denied"), e.getMessage()); + // Unlike a missing group, nothing undoes a REVOKE by itself, so the pause must not be auto + // resumed - otherwise the job flaps every few minutes for as long as the grant is missing. + Assertions.assertEquals(InternalErrorCode.CANNOT_RESUME_ERR, e.getErrorCode()); + } + + // Non-cloud has no named compute group, so the check is skipped entirely. + @Test + public void testRuntimeCheckSkipsComputeGroupInNonCloudMode() throws UserException { + Config.deploy_mode = ""; + Config.cloud_unique_id = ""; + ComputeGroupBindingUtil.checkComputeGroupBeforeTask(UserIdentity.ADMIN, "cg_gone"); + } + + private static class TestCloudSystemInfoService extends CloudSystemInfoService { + private final List clusterNames; + + private TestCloudSystemInfoService(List clusterNames) { + this.clusterNames = clusterNames; + } + + @Override + public List getCloudClusterNames() { + return clusterNames; + } + + // getComputeGroupByName() resolves through these two, so a known name has to look + // resolvable here or the existence check fires before anything else can be exercised. + @Override + public String getPhysicalCluster(String clusterName) { + return clusterName; + } + + @Override + public String getCloudClusterIdByName(String clusterName) { + return clusterNames.contains(clusterName) ? clusterName + "_id" : ""; + } + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_compute_group_binding_isolation.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_compute_group_binding_isolation.groovy new file mode 100644 index 00000000000000..e2323b52cf4550 --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/test_compute_group_binding_isolation.groovy @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions + +// Isolation across compute groups, end to end. +// +// Two async materialized views declaring different compute groups must refresh in their own group +// and nowhere else, no matter which group the triggering session is on. The MV refresh path needs +// no external data source, which makes it the cheapest way to prove the binding end to end. +suite('test_compute_group_binding_isolation', 'multi_cluster,docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.cloudMode = true + + docker(options) { + def secondGroup = "cg_isolation_b" + + // Resolve the session's own compute group before the second group exists. A session with no + // explicit group picks the first group with an alive backend, and that list is sorted by + // name, so once cg_isolation_b is up it would win over the default group and the two names + // below would collide. + def groups = sql_return_maparray """show clusters""" + logger.info("clusters: ${groups}") + def firstGroup = groups.stream() + .filter(cg -> cg.is_current == "TRUE").findFirst().orElse(null)?.cluster + assertNotNull(firstGroup) + assertNotEquals(firstGroup, secondGroup) + + cluster.addBackend(1, secondGroup) + // Keep this session on its original group now that a second one exists. + sql """use @${firstGroup}""" + + def tableName = "test_cg_isolation_tbl" + def mvOnFirst = "test_cg_isolation_mv_a" + def mvOnSecond = "test_cg_isolation_mv_b" + + sql """DROP TABLE IF EXISTS ${tableName} FORCE""" + sql """ + CREATE TABLE ${tableName} ( + `k1` INT NULL, + `k2` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ('replication_num' = '1'); + """ + sql """INSERT INTO ${tableName} VALUES (1, 1), (2, 2), (3, 3);""" + + sql """ + CREATE MATERIALIZED VIEW ${mvOnFirst} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1', 'compute_group' = '${firstGroup}') + AS SELECT k1, k2 FROM ${tableName}; + """ + sql """ + CREATE MATERIALIZED VIEW ${mvOnSecond} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1', 'compute_group' = '${secondGroup}') + AS SELECT k1, k2 FROM ${tableName}; + """ + + sql """REFRESH MATERIALIZED VIEW ${mvOnFirst} AUTO;""" + sql """REFRESH MATERIALIZED VIEW ${mvOnSecond} AUTO;""" + waitingMTMVTaskFinishedByMvName(mvOnFirst) + waitingMTMVTaskFinishedByMvName(mvOnSecond) + + def computeGroupOfLastTask = { mvName -> + def tasks = sql_return_maparray """ + select * from tasks("type"="mv") where MvName = '${mvName}' order by CreateTime desc + """ + assertTrue(tasks.size() > 0) + logger.info("tasks of ${mvName}: ${tasks}") + return tasks.get(0).ComputeGroup + } + + // Each MV refreshed in its own compute group, and the two are different. + assertEquals(firstGroup, computeGroupOfLastTask(mvOnFirst)) + assertEquals(secondGroup, computeGroupOfLastTask(mvOnSecond)) + + // Both produced correct data, i.e. pinning to a non-session group did not break the refresh. + def rowsA = sql "SELECT COUNT(*) FROM ${mvOnFirst}" + def rowsB = sql "SELECT COUNT(*) FROM ${mvOnSecond}" + assertEquals(3, rowsA.get(0).get(0)) + assertEquals(3, rowsB.get(0).get(0)) + + // The declaration wins over the triggering session: refreshing from inside the second group + // must still send the first MV to the first group. + // COMPLETE, not AUTO: the base table has not changed since the first refresh, so AUTO would + // resolve to NOT_REFRESH and never report a compute group to assert on. + sql """use @${secondGroup}""" + sql """REFRESH MATERIALIZED VIEW ${mvOnFirst} COMPLETE;""" + waitingMTMVTaskFinishedByMvName(mvOnFirst) + assertEquals(firstGroup, computeGroupOfLastTask(mvOnFirst)) + + // And an MV that declares nothing keeps the old behaviour of borrowing the session's group. + def mvUndeclared = "test_cg_isolation_mv_none" + sql """ + CREATE MATERIALIZED VIEW ${mvUndeclared} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS SELECT k1, k2 FROM ${tableName}; + """ + sql """REFRESH MATERIALIZED VIEW ${mvUndeclared} AUTO;""" + waitingMTMVTaskFinishedByMvName(mvUndeclared) + assertEquals(secondGroup, computeGroupOfLastTask(mvUndeclared)) + + sql """use @${firstGroup}""" + sql """DROP MATERIALIZED VIEW IF EXISTS ${mvOnFirst};""" + sql """DROP MATERIALIZED VIEW IF EXISTS ${mvOnSecond};""" + sql """DROP MATERIALIZED VIEW IF EXISTS ${mvUndeclared};""" + sql """DROP TABLE IF EXISTS ${tableName} FORCE""" + } +} diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_compute_group.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_compute_group.groovy new file mode 100644 index 00000000000000..82b5c42a0ae6ba --- /dev/null +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_compute_group.groovy @@ -0,0 +1,292 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.clients.producer.ProducerConfig + +// Declaring `compute_group` on a routine load job. +// +// The declaration is stored in the job's property map under the key `compute_group`, which is the +// same key and value space the final (owner, compute_group, workload_group) design uses, so the +// metadata is read back as an explicit pin by later versions without conversion. +// +// The property-level checks run before the Kafka data source is touched, so the negative cases do +// not need a broker; only the "job really runs there" cases do. +suite("test_routine_load_compute_group", "p0") { + String tableName = "test_routine_load_compute_group_tbl" + String jobName = "test_routine_load_compute_group_job" + + sql """DROP TABLE IF EXISTS ${tableName}""" + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` INT NULL, + `k2` STRING NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + + // Returns the statement instead of running it: inside a `test { }` block the `sql` method must + // be the action's own, so the SQL has to be handed to it there rather than executed here. + def createJobSql = { String computeGroup -> + return """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS TERMINATED BY "," + PROPERTIES ("compute_group" = "${computeGroup}") + FROM KAFKA ("kafka_broker_list" = "127.0.0.1:19092", "kafka_topic" = "unused_topic"); + """.toString() + } + + if (!isCloudMode()) { + // Non-cloud is out of scope for this transitional change, so the key can never appear in + // non-cloud metadata and upgrading such a cluster has nothing to convert. The cloud-mode + // check runs first, so even the reserved DEFAULT is refused with this message. + for (String value : ["any_group", "DEFAULT", "default"]) { + test { + sql createJobSql(value) + exception "only supported in cloud mode" + } + } + sql """DROP TABLE IF EXISTS ${tableName}""" + return + } + + // ---------------- cloud mode ---------------- + + // DEFAULT is reserved by the final design ("follow the owner's default group"). A job pinned to + // a group literally named DEFAULT would be silently reinterpreted after an upgrade. + for (String reserved : ["DEFAULT", "default", "Default"]) { + test { + sql createJobSql(reserved) + exception "reserved value" + } + } + + test { + sql createJobSql("cg_that_does_not_exist") + exception "not found" + } + + def currentComputeGroup = sql_return_maparray("show clusters") + .stream().filter(cg -> cg.is_current == "TRUE").findFirst().orElse(null) + assertNotNull(currentComputeGroup) + def cgName = currentComputeGroup.cluster + logger.info("current compute group: ${cgName}") + + String enabled = context.config.otherConfigs.get("enableKafkaTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("kafka test not enabled, skipping the running-job part") + sql """DROP TABLE IF EXISTS ${tableName}""" + return + } + + String kafka_port = context.config.otherConfigs.get("kafka_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def kafka_broker = "${externalEnvIp}:${kafka_port}" + // A topic of its own on each run: the job reads from OFFSET_BEGINNING and the count below is exact, so + // records left by an earlier run against the same broker would be counted too. + def topic = "test_routine_load_compute_group_${UUID.randomUUID()}".toString() + + def props = new Properties() + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringSerializer") + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringSerializer") + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + def producer = new KafkaProducer<>(props) + for (int i = 0; i < 5; i++) { + producer.send(new ProducerRecord<>(topic, "${i},row${i}".toString())) + } + producer.close() + + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS TERMINATED BY "," + PROPERTIES ( + "max_batch_interval" = "5", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200", + "compute_group" = "${cgName}" + ) + FROM KAFKA ( + "kafka_broker_list" = "${kafka_broker}", + "kafka_topic" = "${topic}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + + // SHOW must report the compute group the job actually runs in, and the declaration must be + // visible in the job properties so operators can inventory pinned jobs before an upgrade. + def show = sql_return_maparray("SHOW ROUTINE LOAD FOR ${jobName}").get(0) + logger.info("show routine load: ${show}") + assertEquals(cgName, show.ComputeGroup) + assertTrue(show.JobProperties.contains("compute_group")) + assertTrue(show.JobProperties.contains(cgName)) + + // The job runs, i.e. pinning did not break the load path. + def count = 0 + while (count < 60) { + def state = sql_return_maparray("SHOW ROUTINE LOAD FOR ${jobName}").get(0).State + def rows = sql "SELECT COUNT(*) FROM ${tableName}" + logger.info("state=${state}, rows=${rows}") + assertNotEquals("PAUSED", state) + if (rows.get(0).get(0) >= 5) { + break + } + sleep(2000) + count++ + } + assertEquals(5, sql("SELECT COUNT(*) FROM ${tableName}").get(0).get(0)) + + // ALTER validates the new value immediately instead of letting a bad name pause the job on + // the next task. ALTER ROUTINE LOAD only accepts PAUSED jobs. + sql "PAUSE ROUTINE LOAD FOR ${jobName}" + test { + sql """ALTER ROUTINE LOAD FOR ${jobName} PROPERTIES("compute_group" = "DEFAULT");""" + exception "reserved value" + } + test { + sql """ALTER ROUTINE LOAD FOR ${jobName} PROPERTIES("compute_group" = "cg_nope");""" + exception "not found" + } + + sql """ALTER ROUTINE LOAD FOR ${jobName} PROPERTIES("compute_group" = "${cgName}");""" + def showAfterAlter = sql_return_maparray("SHOW ROUTINE LOAD FOR ${jobName}").get(0) + assertEquals(cgName, showAfterAlter.ComputeGroup) + // SHOW CREATE carries the declaration, so a job recreated from its output is pinned the same way. + def showCreate = sql_return_maparray("SHOW CREATE ROUTINE LOAD FOR ${jobName}").get(0) + logger.info("show create routine load: ${showCreate}") + assertTrue(showCreate.CreateStmt.contains("\"compute_group\" = \"${cgName}\"")) + + sql "RESUME ROUTINE LOAD FOR ${jobName}" + } finally { + try { + sql "STOP ROUTINE LOAD FOR ${jobName}" + } catch (Exception e) { + logger.info("stop routine load failed: ${e.getMessage()}") + } + sql """DROP TABLE IF EXISTS ${tableName}""" + } + + // The binding is re-checked before every task, not only at CREATE time: revoking the job + // owner's USAGE on the declared compute group must pause the job with that reason. + // + // The check is deliberately placed before the task takes any resource - before backend + // allocation and before the load transaction is begun - so that a group which has been dropped + // outright cannot first surface as "no available BE found". That ordering is what + // RoutineLoadComputeGroupTest#testDroppedComputeGroupFailsTaskBeforeBackendAllocation pins + // down; here we cover the privilege half end to end, which needs a real running job. + String revokeUser = "test_rl_cg_revoke_user" + String revokePwd = "Test_12345" + String revokeJob = "test_rl_cg_revoke_job" + String revokeTable = "test_rl_cg_revoke_tbl" + + sql """DROP TABLE IF EXISTS ${revokeTable}""" + sql """ + CREATE TABLE ${revokeTable} ( + `k1` INT NULL, + `k2` STRING NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + sql """DROP USER IF EXISTS ${revokeUser}""" + sql """CREATE USER '${revokeUser}' IDENTIFIED BY '${revokePwd}'""" + sql """GRANT LOAD_PRIV, SELECT_PRIV ON *.*.* TO ${revokeUser}""" + sql """GRANT USAGE_PRIV ON COMPUTE GROUP `${cgName}` TO ${revokeUser}""" + + try { + // Created by the user, so RoutineLoadJob#userIdentity - the identity the per task check + // runs against - is that user rather than the admin running this suite. + connect(revokeUser, "${revokePwd}", context.config.jdbcUrl) { + sql """use ${context.dbName}""" + sql """ + CREATE ROUTINE LOAD ${revokeJob} ON ${revokeTable} + COLUMNS TERMINATED BY "," + PROPERTIES ( + "max_batch_interval" = "5", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200", + "compute_group" = "${cgName}" + ) + FROM KAFKA ( + "kafka_broker_list" = "${kafka_broker}", + "kafka_topic" = "${topic}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + } + + def runningBeforeRevoke = false + for (int i = 0; i < 60; i++) { + def state = sql_return_maparray("SHOW ROUTINE LOAD FOR ${revokeJob}").get(0).State + if (state == "RUNNING") { + runningBeforeRevoke = true + break + } + sleep(1000) + } + assertTrue(runningBeforeRevoke, "the job must be running before its privilege is revoked") + + sql """REVOKE USAGE_PRIV ON COMPUTE GROUP `${cgName}` FROM ${revokeUser}""" + + // The check runs ahead of the "is there more data to consume" probe, so the job is refused + // on the next scheduling round whether or not the topic has new records. + def pausedForComputeGroup = false + def sawBeAvailabilityReason = false + def lastSeen = "" + for (int i = 0; i < 90; i++) { + def row = sql_return_maparray("SHOW ROUTINE LOAD FOR ${revokeJob}").get(0) + def reason = row.ReasonOfStateChanged == null ? "" : row.ReasonOfStateChanged + lastSeen = "state=${row.State}, reason=${reason}" + if (reason.contains("no available BE found")) { + sawBeAvailabilityReason = true + } + if (reason.contains("USAGE denied") && reason.contains(cgName)) { + pausedForComputeGroup = true + break + } + sleep(1000) + } + assertTrue(pausedForComputeGroup, + "revoking USAGE must fail the next task with the real reason, last seen: ${lastSeen}".toString()) + assertFalse(sawBeAvailabilityReason, + "a compute group problem must never be reported as a BE availability problem") + + // Nothing undoes a REVOKE by itself, so the pause carries CANNOT_RESUME_ERR and the job + // must stay down instead of being auto resumed into the same failure every few minutes. + for (int i = 0; i < 15; i++) { + def state = sql_return_maparray("SHOW ROUTINE LOAD FOR ${revokeJob}").get(0).State + assertEquals("PAUSED", state, + "a job paused by a revoked privilege must not be auto resumed") + sleep(1000) + } + } finally { + try { + sql "STOP ROUTINE LOAD FOR ${revokeJob}" + } catch (Exception e) { + logger.info("stop routine load failed: ${e.getMessage()}") + } + sql """DROP TABLE IF EXISTS ${revokeTable}""" + sql """DROP USER IF EXISTS ${revokeUser}""" + } +} diff --git a/regression-test/suites/mtmv_p0/test_mtmv_compute_group.groovy b/regression-test/suites/mtmv_p0/test_mtmv_compute_group.groovy new file mode 100644 index 00000000000000..a6380c496d22c8 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_mtmv_compute_group.groovy @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Declaring `compute_group` on an async materialized view. +// +// The property is a transitional binding whose name and value space match the final +// (owner, compute_group, workload_group) design, so metadata written here is read as an explicit +// pin by later versions. Two values are rejected on purpose: anything in non-cloud mode, and the +// reserved word DEFAULT. The non-cloud check runs first, so in non-cloud mode every value - DEFAULT +// included - is refused with the same "cloud mode" message. +suite("test_mtmv_compute_group") { + String suiteName = "test_mtmv_compute_group" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + + sql """drop materialized view if exists ${mvName};""" + sql """drop table if exists `${tableName}`""" + + sql """ + CREATE TABLE `${tableName}` ( + `k1` INT NULL, + `k2` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ('replication_num' = '1'); + """ + sql """insert into ${tableName} values(1, 1), (2, 2);""" + + // Returns the statement instead of running it: inside a `test { }` block the `sql` method must + // be the action's own, so the SQL has to be handed to it there rather than executed here. + def createMvSql = { String properties -> + return """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES (${properties}) + AS SELECT k1, k2 FROM ${tableName}; + """.toString() + } + + if (!isCloudMode()) { + // Non-cloud is out of scope for this transitional change: declaring the property must be + // rejected whatever the value, so that non-cloud metadata never carries the key and + // upgrading such a cluster has nothing to convert. + for (String value : ["any_group", "DEFAULT", "default"]) { + test { + sql createMvSql("'replication_num' = '1', 'compute_group' = '${value}'") + exception "only supported in cloud mode" + } + } + + // An MV without the property must keep working exactly as before. + sql createMvSql("'replication_num' = '1'") + sql """REFRESH MATERIALIZED VIEW ${mvName} AUTO;""" + waitingMTMVTaskFinishedByMvName(mvName) + assertEquals(2, sql("SELECT COUNT(*) FROM ${mvName}").get(0).get(0)) + + // ALTER must reject it too, not only CREATE. + test { + sql """ALTER MATERIALIZED VIEW ${mvName} SET ('compute_group' = 'any_group');""" + exception "only supported in cloud mode" + } + + sql """drop materialized view if exists ${mvName};""" + sql """drop table if exists `${tableName}`""" + return + } + + // ---------------- cloud mode ---------------- + + // DEFAULT is reserved by the final design ("follow the owner's default group"). Pinning a group + // literally named DEFAULT would be silently reinterpreted after an upgrade. + for (String reserved : ["DEFAULT", "default", "Default"]) { + test { + sql createMvSql("'replication_num' = '1', 'compute_group' = '${reserved}'") + exception "reserved value" + } + } + + test { + sql createMvSql("'replication_num' = '1', 'compute_group' = 'cg_that_does_not_exist'") + exception "not found" + } + + def currentComputeGroup = sql_return_maparray("show clusters") + .stream().filter(cg -> cg.is_current == "TRUE").findFirst().orElse(null) + assertNotNull(currentComputeGroup) + def cgName = currentComputeGroup.cluster + logger.info("current compute group: ${cgName}") + + sql createMvSql("'replication_num' = '1', 'compute_group' = '${cgName}'") + + // The declaration is persisted on the MV and visible. + def showCreate = sql """show create materialized view ${mvName};""" + assertTrue(showCreate.toString().contains("compute_group")) + assertTrue(showCreate.toString().contains(cgName)) + + sql """REFRESH MATERIALIZED VIEW ${mvName} AUTO;""" + waitingMTMVTaskFinishedByMvName(mvName) + assertEquals(2, sql("SELECT COUNT(*) FROM ${mvName}").get(0).get(0)) + + // The refresh really ran in the declared compute group. + def tasks = sql_return_maparray """select * from tasks("type"="mv") where MvName = '${mvName}'""" + assertTrue(tasks.size() > 0) + logger.info("mv tasks: ${tasks}") + assertEquals(cgName, tasks.get(0).ComputeGroup) + + // ALTER keeps the same value space, including the DEFAULT rejection. + test { + sql """ALTER MATERIALIZED VIEW ${mvName} SET ('compute_group' = 'DEFAULT');""" + exception "reserved value" + } + test { + sql """ALTER MATERIALIZED VIEW ${mvName} SET ('compute_group' = 'cg_that_does_not_exist');""" + exception "not found" + } + + // Re-declaring the same group is a no-op and must keep refreshing there. The new row gives the AUTO + // refresh something to do: with an unchanged base table it is NOT_REFRESH, which ends before a + // compute group is applied and records none. The task checked is the one this refresh created, + // found by id, because tasks() is not ordered and ids created in the same second are not monotonic. + def taskIdsBeforeAlter = sql_return_maparray("""select TaskId from tasks("type"="mv") where MvName = '${mvName}'""") + .collect { it.TaskId } + sql """ALTER MATERIALIZED VIEW ${mvName} SET ('compute_group' = '${cgName}');""" + sql """insert into ${tableName} values(3, 3);""" + sql """REFRESH MATERIALIZED VIEW ${mvName} AUTO;""" + waitingMTMVTaskFinishedByMvName(mvName) + assertEquals(3, sql("SELECT COUNT(*) FROM ${mvName}").get(0).get(0)) + def tasksAfterAlter = sql_return_maparray("""select * from tasks("type"="mv") where MvName = '${mvName}'""") + .findAll { !taskIdsBeforeAlter.contains(it.TaskId) } + logger.info("mv tasks after alter: ${tasksAfterAlter}") + assertEquals(1, tasksAfterAlter.size()) + assertEquals(cgName, tasksAfterAlter.get(0).ComputeGroup) + + sql """drop materialized view if exists ${mvName};""" + sql """drop table if exists `${tableName}`""" +}