Repository navigation
[Feature]bind workload group to compute group #49884
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -35,6 +35,9 @@ | |
| import com.google.common.collect.Maps; | ||
| import lombok.Getter; | ||
| import org.apache.commons.collections.MapUtils; | ||
| import org.apache.commons.lang3.StringUtils; | ||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
|
|
||
| import java.util.Map; | ||
| import java.util.Optional; | ||
|
|
@@ -50,6 +53,8 @@ | |
| */ | ||
| public class AlterRoutineLoadStmt extends DdlStmt implements NotFallbackInParser { | ||
|
|
||
| private static final Logger LOG = LogManager.getLogger(AlterRoutineLoadStmt.class); | ||
|
|
||
| private static final String NAME_TYPE = "ROUTINE LOAD NAME"; | ||
|
|
||
| private static final ImmutableSet<String> CONFIGURABLE_JOB_PROPERTIES_SET = new ImmutableSet.Builder<String>() | ||
|
|
@@ -248,9 +253,24 @@ private void checkJobProperties() throws UserException { | |
| } | ||
| if (jobProperties.containsKey(CreateRoutineLoadStmt.WORKLOAD_GROUP)) { | ||
| String workloadGroup = jobProperties.get(CreateRoutineLoadStmt.WORKLOAD_GROUP); | ||
| long wgId = Env.getCurrentEnv().getWorkloadGroupMgr() | ||
| .getWorkloadGroup(ConnectContext.get().getCurrentUserIdentity(), workloadGroup); | ||
| analyzedJobProperties.put(CreateRoutineLoadStmt.WORKLOAD_GROUP, String.valueOf(wgId)); | ||
| if (!StringUtils.isEmpty(workloadGroup)) { | ||
| ConnectContext tmpCtx = new ConnectContext(); | ||
| tmpCtx.setCurrentUserIdentity(ConnectContext.get().getCurrentUserIdentity()); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. need add comment to explain why get 0 index |
||
| tmpCtx.getSessionVariable().setWorkloadGroup(workloadGroup); | ||
| long wgId; | ||
| try { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 如果workload group id 不存在,也得让他能跑。 因为我们重启的时候,会drop old workload gorup 然后重新建立,此时routine load 可能就跑不了了。 |
||
| // NOTE(wb): why get 0th wg here; | ||
| // currently a routineload can only be executed in one workload group; | ||
| // but the workload group thrift sent from FE to BE is a list, this is for scalability. | ||
| wgId = Env.getCurrentEnv().getWorkloadGroupMgr() | ||
| .getWorkloadGroup(tmpCtx).get(0) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这个get 方法为什么返回的是一个list? |
||
| .getId(); | ||
| } catch (Throwable t) { | ||
| LOG.info("Get workload group failed when alter routine load,", t); | ||
| throw t; | ||
| } | ||
| analyzedJobProperties.put(CreateRoutineLoadStmt.WORKLOAD_GROUP, String.valueOf(wgId)); | ||
| } | ||
| } | ||
| if (jobProperties.containsKey(LoadStmt.KEY_ENCLOSE)) { | ||
| analyzedJobProperties.put(LoadStmt.KEY_ENCLOSE, jobProperties.get(LoadStmt.KEY_ENCLOSE)); | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -494,7 +494,7 @@ public class SchemaTable extends Table { | |
| .column("MIN_REMOTE_SCAN_THREAD_NUM", ScalarType.createType(PrimitiveType.BIGINT)) | ||
| .column("MEMORY_LOW_WATERMARK", ScalarType.createVarchar(256)) | ||
| .column("MEMORY_HIGH_WATERMARK", ScalarType.createVarchar(256)) | ||
| .column("TAG", ScalarType.createVarchar(256)) | ||
| .column("COMPUTE_GROUP", ScalarType.createVarchar(256)) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这些地方,我们得记得改文档。
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这么说的话,要改的不止一处啊,所有报错的地方,都得是compute_group/resource group.... |
||
| .column("READ_BYTES_PER_SECOND", ScalarType.createType(PrimitiveType.BIGINT)) | ||
| .column("REMOTE_READ_BYTES_PER_SECOND", ScalarType.createType(PrimitiveType.BIGINT)) | ||
| .build())) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -147,8 +147,13 @@ public long createLoadJobFromStmt(LoadStmt stmt) throws DdlException, UserExcept | |
| } | ||
|
|
||
| if (Config.enable_workload_group) { | ||
| loadJob.settWorkloadGroups( | ||
| Env.getCurrentEnv().getWorkloadGroupMgr().getWorkloadGroup(ConnectContext.get())); | ||
| try { | ||
| loadJob.settWorkloadGroups( | ||
| Env.getCurrentEnv().getWorkloadGroupMgr().getWorkloadGroup(ConnectContext.get())); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 我们这为啥都是list |
||
| } catch (Throwable t) { | ||
| LOG.info("Get workload group failed when create load job,", t); | ||
| throw t; | ||
| } | ||
| } | ||
|
|
||
| Env.getCurrentEnv().getEditLog().logCreateLoadJob(loadJob); | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,10 +21,12 @@ | |
| import org.apache.doris.catalog.Env; | ||
| import org.apache.doris.catalog.OlapTable; | ||
| import org.apache.doris.catalog.Table; | ||
| import org.apache.doris.cloud.system.CloudSystemInfoService; | ||
| import org.apache.doris.common.Config; | ||
| import org.apache.doris.common.UserException; | ||
| import org.apache.doris.common.util.DebugUtil; | ||
| import org.apache.doris.planner.StreamLoadPlanner; | ||
| import org.apache.doris.qe.ConnectContext; | ||
| import org.apache.doris.thrift.TFileFormatType; | ||
| import org.apache.doris.thrift.TKafkaLoadInfo; | ||
| import org.apache.doris.thrift.TLoadSourceType; | ||
|
|
@@ -35,7 +37,10 @@ | |
| import org.apache.doris.thrift.TUniqueId; | ||
|
|
||
| import com.google.common.base.Joiner; | ||
| import com.google.common.base.Strings; | ||
| import com.google.gson.Gson; | ||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
@@ -45,6 +50,8 @@ | |
| public class KafkaTaskInfo extends RoutineLoadTaskInfo { | ||
| private RoutineLoadManager routineLoadManager = Env.getCurrentEnv().getRoutineLoadManager(); | ||
|
|
||
| private static final Logger LOG = LogManager.getLogger(KafkaTaskInfo.class); | ||
|
|
||
| // <partitionId, offset to be consumed> | ||
| private Map<Integer, Long> partitionIdToOffset; | ||
|
|
||
|
|
@@ -138,21 +145,39 @@ private TPipelineFragmentParams rePlan(RoutineLoadJob routineLoadJob) throws Use | |
| tPlanFragment.getOutputSink().getOlapTableSink().setTxnId(txnId); | ||
|
|
||
| if (Config.enable_workload_group) { | ||
| long wgId = routineLoadJob.getWorkloadId(); | ||
| List<TPipelineWorkloadGroup> tWgList = new ArrayList<>(); | ||
| if (wgId > 0) { | ||
| tWgList = Env.getCurrentEnv().getWorkloadGroupMgr() | ||
| .getTWorkloadGroupById(wgId); | ||
| try { | ||
| long wgId = routineLoadJob.getWorkloadId(); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 为啥叫 getWorkloadId? |
||
| List<TPipelineWorkloadGroup> tWgList = new ArrayList<>(); | ||
| if (wgId > 0) { | ||
| tWgList = Env.getCurrentEnv().getWorkloadGroupMgr() | ||
| .getTWorkloadGroupById(wgId); | ||
| } | ||
|
|
||
| if (tWgList.size() == 0) { | ||
| throw new UserException("can not find workload group, id=" + wgId); | ||
| ConnectContext tmpContext = new ConnectContext(); | ||
| if (Config.isCloudMode()) { | ||
| String clusterName = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) | ||
| .getClusterNameByClusterId(routineLoadJob.getCloudClusterId()); | ||
| if (Strings.isNullOrEmpty(clusterName)) { | ||
| String err = String.format("cluster name is empty, cluster id is %s", | ||
| routineLoadJob.getCloudClusterId()); | ||
| LOG.warn(err); | ||
| throw new UserException(err); | ||
| } | ||
| tmpContext.setCloudCluster(clusterName); | ||
| } | ||
| tmpContext.setCurrentUserIdentity(routineLoadJob.getUserIdentity()); | ||
| tmpContext.setQualifiedUser(routineLoadJob.getUserIdentity().getQualifiedUser()); | ||
| tWgList = Env.getCurrentEnv().getWorkloadGroupMgr().getWorkloadGroup(tmpContext); | ||
| } | ||
| } else { | ||
| tWgList = Env.getCurrentEnv().getWorkloadGroupMgr() | ||
| .getWorkloadGroupByUser(routineLoadJob.getUserIdentity(), false); | ||
| } | ||
| if (tWgList.size() != 0) { | ||
| tExecPlanFragmentParams.setWorkloadGroups(tWgList); | ||
| if (tWgList.size() != 0) { | ||
| tExecPlanFragmentParams.setWorkloadGroups(tWgList); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 为啥是list? |
||
| } | ||
| } catch (Throwable t) { | ||
| LOG.info("Get workload group failed when replan kafka, job id:{} , ", routineLoadJob.getTxnId(), t); | ||
| throw t; | ||
| } | ||
|
|
||
| } | ||
|
|
||
| return tExecPlanFragmentParams; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5301,9 +5301,10 @@ public LogicalPlan visitShowCreateMaterializedView(ShowCreateMaterializedViewCon | |
|
|
||
| @Override | ||
| public LogicalPlan visitAlterWorkloadGroup(AlterWorkloadGroupContext ctx) { | ||
| String cgName = ctx.computeGroup == null ? "" : stripQuotes(ctx.computeGroup.getText()); | ||
| Map<String, String> properties = ctx.propertyClause() != null | ||
| ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap(); | ||
| return new AlterWorkloadGroupCommand(ctx.name.getText(), properties); | ||
| return new AlterWorkloadGroupCommand(cgName, ctx.name.getText(), properties); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 不要传空。 传递default |
||
| } | ||
|
|
||
| @Override | ||
|
|
@@ -5585,10 +5586,11 @@ public LogicalPlan visitDropEncryptkey(DropEncryptkeyContext ctx) { | |
| @Override | ||
| public LogicalPlan visitCreateWorkloadGroup(CreateWorkloadGroupContext ctx) { | ||
| String workloadGroupName = stripQuotes(ctx.name.getText()); | ||
| String cgName = ctx.computeGroup == null ? "" : stripQuotes(ctx.computeGroup.getText()); | ||
| boolean ifNotExists = ctx.EXISTS() != null; | ||
| Map<String, String> properties = ctx.propertyClause() != null | ||
| ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap(); | ||
| return new CreateWorkloadGroupCommand(workloadGroupName, ifNotExists, properties); | ||
| return new CreateWorkloadGroupCommand(cgName, workloadGroupName, ifNotExists, properties); | ||
| } | ||
|
|
||
| @Override | ||
|
|
@@ -5630,7 +5632,8 @@ public LogicalPlan visitDropUser(DropUserContext ctx) { | |
|
|
||
| @Override | ||
| public LogicalPlan visitDropWorkloadGroup(DropWorkloadGroupContext ctx) { | ||
| return new DropWorkloadGroupCommand(ctx.name.getText(), ctx.EXISTS() != null); | ||
| String cgName = ctx.computeGroup == null ? "" : stripQuotes(ctx.computeGroup.getText()); | ||
| return new DropWorkloadGroupCommand(cgName, ctx.name.getText(), ctx.EXISTS() != null); | ||
| } | ||
|
|
||
| @Override | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
至少要增加一个SQL BASED regression test,来保证这个语法正确,包括对应的系统表