Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ std::vector<SchemaScanner::ColumnDesc> SchemaWorkloadGroupsScanner::_s_tbls_colu
{"MIN_REMOTE_SCAN_THREAD_NUM", TYPE_BIGINT, sizeof(int64_t), true},
{"MEMORY_LOW_WATERMARK", TYPE_VARCHAR, sizeof(StringRef), true},
{"MEMORY_HIGH_WATERMARK", TYPE_VARCHAR, sizeof(StringRef), true},
{"TAG", TYPE_VARCHAR, sizeof(StringRef), true},
{"COMPUTE_GROUP", TYPE_VARCHAR, sizeof(StringRef), true},
{"READ_BYTES_PER_SECOND", TYPE_BIGINT, sizeof(int64_t), true},
{"REMOTE_READ_BYTES_PER_SECOND", TYPE_BIGINT, sizeof(int64_t), true},
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1986,10 +1986,6 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static boolean enable_query_queue = true;

// used for regression test
@ConfField(mutable = true)
public static boolean enable_alter_queue_prop_sync = false;

@ConfField(mutable = true)
public static long query_queue_update_interval_ms = 5000;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ supportedCreateStatement
(WITH ROLLUP (rollupNames=identifierList)?)? #createTableLike
| CREATE ROLE (IF NOT EXISTS)? name=identifierOrText (COMMENT STRING_LITERAL)? #createRole
| CREATE WORKLOAD GROUP (IF NOT EXISTS)?
name=identifierOrText properties=propertyClause? #createWorkloadGroup
name=identifierOrText (FOR computeGroup=identifierOrText)? properties=propertyClause? #createWorkloadGroup

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

至少要增加一个SQL BASED regression test,来保证这个语法正确,包括对应的系统表

| CREATE CATALOG (IF NOT EXISTS)? catalogName=identifier
(WITH RESOURCE resourceName=identifier)?
(COMMENT STRING_LITERAL)? properties=propertyClause? #createCatalog
Expand Down Expand Up @@ -226,7 +226,7 @@ supportedAlterStatement
| ALTER CATALOG name=identifier RENAME newName=identifier #alterCatalogRename
| ALTER ROLE role=identifierOrText commentSpec #alterRole
| ALTER STORAGE VAULT name=multipartIdentifier properties=propertyClause #alterStorageVault
| ALTER WORKLOAD GROUP name=identifierOrText
| ALTER WORKLOAD GROUP name=identifierOrText (FOR computeGroup=identifierOrText)?
properties=propertyClause? #alterWorkloadGroup
| ALTER CATALOG name=identifier SET PROPERTIES
LEFT_PAREN propertyItemList RIGHT_PAREN #alterCatalogProperties
Expand Down Expand Up @@ -258,7 +258,7 @@ supportedDropStatement
| DROP SQL_BLOCK_RULE (IF EXISTS)? identifierSeq #dropSqlBlockRule
| DROP USER (IF EXISTS)? userIdentify #dropUser
| DROP STORAGE POLICY (IF EXISTS)? name=identifier #dropStoragePolicy
| DROP WORKLOAD GROUP (IF EXISTS)? name=identifierOrText #dropWorkloadGroup
| DROP WORKLOAD GROUP (IF EXISTS)? name=identifierOrText (FOR computeGroup=identifierOrText)? #dropWorkloadGroup
| DROP CATALOG (IF EXISTS)? name=identifier #dropCatalog
| DROP FILE name=STRING_LITERAL
((FROM | IN) database=identifier)? properties=propertyClause #dropFile
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@
import com.google.common.collect.Maps;
import lombok.Getter;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.Map;
import java.util.Optional;
Expand All @@ -50,6 +53,8 @@
*/
public class AlterRoutineLoadStmt extends DdlStmt implements NotFallbackInParser {

private static final Logger LOG = LogManager.getLogger(AlterRoutineLoadStmt.class);

private static final String NAME_TYPE = "ROUTINE LOAD NAME";

private static final ImmutableSet<String> CONFIGURABLE_JOB_PROPERTIES_SET = new ImmutableSet.Builder<String>()
Expand Down Expand Up @@ -248,9 +253,24 @@ private void checkJobProperties() throws UserException {
}
if (jobProperties.containsKey(CreateRoutineLoadStmt.WORKLOAD_GROUP)) {
String workloadGroup = jobProperties.get(CreateRoutineLoadStmt.WORKLOAD_GROUP);
long wgId = Env.getCurrentEnv().getWorkloadGroupMgr()
.getWorkloadGroup(ConnectContext.get().getCurrentUserIdentity(), workloadGroup);
analyzedJobProperties.put(CreateRoutineLoadStmt.WORKLOAD_GROUP, String.valueOf(wgId));
if (!StringUtils.isEmpty(workloadGroup)) {
ConnectContext tmpCtx = new ConnectContext();
tmpCtx.setCurrentUserIdentity(ConnectContext.get().getCurrentUserIdentity());

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

need add comment to explain why get 0 index

tmpCtx.getSessionVariable().setWorkloadGroup(workloadGroup);
long wgId;
try {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

如果workload group id 不存在,也得让他能跑。 因为我们重启的时候,会drop old workload gorup 然后重新建立,此时routine load 可能就跑不了了。

// NOTE(wb): why get 0th wg here;
// currently a routineload can only be executed in one workload group;
// but the workload group thrift sent from FE to BE is a list, this is for scalability.
wgId = Env.getCurrentEnv().getWorkloadGroupMgr()
.getWorkloadGroup(tmpCtx).get(0)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个get 方法为什么返回的是一个list?

.getId();
} catch (Throwable t) {
LOG.info("Get workload group failed when alter routine load,", t);
throw t;
}
analyzedJobProperties.put(CreateRoutineLoadStmt.WORKLOAD_GROUP, String.valueOf(wgId));
}
}
if (jobProperties.containsKey(LoadStmt.KEY_ENCLOSE)) {
analyzedJobProperties.put(LoadStmt.KEY_ENCLOSE, jobProperties.get(LoadStmt.KEY_ENCLOSE));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -582,8 +582,20 @@ private void checkJobProperties() throws UserException {

String inputWorkloadGroupStr = jobProperties.get(WORKLOAD_GROUP);
if (!StringUtils.isEmpty(inputWorkloadGroupStr)) {
this.workloadGroupId = Env.getCurrentEnv().getWorkloadGroupMgr()
.getWorkloadGroup(ConnectContext.get().getCurrentUserIdentity(), inputWorkloadGroupStr);
ConnectContext tmpCtx = new ConnectContext();
tmpCtx.setCurrentUserIdentity(ConnectContext.get().getCurrentUserIdentity());
tmpCtx.getSessionVariable().setWorkloadGroup(inputWorkloadGroupStr);
try {
// NOTE(wb): why get 0th wg here;
// currently a routineload can only be executed in one workload group;
// but the workload group thrift sent from FE to BE is a list, this is for scalability.
this.workloadGroupId = Env.getCurrentEnv().getWorkloadGroupMgr()
.getWorkloadGroup(tmpCtx).get(0)
.getId();
} catch (Throwable t) {
LOG.info("Get workload group failed when create routine load,", t);
throw t;
}
}

if (ConnectContext.get() != null) {
Expand Down
3 changes: 3 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,7 @@
import org.apache.doris.resource.AdmissionControl;
import org.apache.doris.resource.Tag;
import org.apache.doris.resource.computegroup.ComputeGroupMgr;
import org.apache.doris.resource.workloadgroup.BindWgToComputeGroupThread;
import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr;
import org.apache.doris.resource.workloadschedpolicy.WorkloadRuntimeStatusMgr;
import org.apache.doris.resource.workloadschedpolicy.WorkloadSchedPolicyMgr;
Expand Down Expand Up @@ -1952,6 +1953,8 @@ protected void startMasterOnlyDaemonThreads() {
topicPublisherThread.addToTopicPublisherList(wpPublisher);
topicPublisherThread.start();

new BindWgToComputeGroupThread().start();

// auto analyze related threads.
statisticsCleaner.start();
statisticsAutoCollector.start();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ public class SchemaTable extends Table {
.column("MIN_REMOTE_SCAN_THREAD_NUM", ScalarType.createType(PrimitiveType.BIGINT))
.column("MEMORY_LOW_WATERMARK", ScalarType.createVarchar(256))
.column("MEMORY_HIGH_WATERMARK", ScalarType.createVarchar(256))
.column("TAG", ScalarType.createVarchar(256))
.column("COMPUTE_GROUP", ScalarType.createVarchar(256))

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这些地方,我们得记得改文档。
另外,这里叫 Compute_GROUP/RESOURCEGROUP 好一些,否则,存算一体下,不好解释

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这么说的话,要改的不止一处啊,所有报错的地方,都得是compute_group/resource group....

.column("READ_BYTES_PER_SECOND", ScalarType.createType(PrimitiveType.BIGINT))
.column("REMOTE_READ_BYTES_PER_SECOND", ScalarType.createType(PrimitiveType.BIGINT))
.build()))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,7 @@ public class FeConstants {
// set to false to disable internal schema db
public static boolean enableInternalSchemaDb = true;

// for UT, create internal workload group thread can not start
public static boolean shouldCreateInternalWorkloadGroup = true;
public static boolean bindWgToComputeGroup = true;

// default scheduler interval is 10 seconds
public static int default_scheduler_interval_millisecond = 10000;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;

public class TopicPublisherThread extends MasterDaemon {
Expand Down Expand Up @@ -143,14 +142,13 @@ public void run() {
Map<TTopicInfoType, List<TopicInfo>> topicMap = copiedRequest.getTopicMap();
List<TopicInfo> topicInfoList = topicMap.get(TTopicInfoType.WORKLOAD_GROUP);
if (topicInfoList != null) {
Set<String> beTagSet = be.getBeWorkloadGroupTagSet();
String beComputeGroup = be.getComputeGroup();
Iterator<TopicInfo> topicIter = topicInfoList.iterator();
while (topicIter.hasNext()) {
TopicInfo topicInfo = topicIter.next();
if (topicInfo.isSetWorkloadGroupInfo()) {
TWorkloadGroupInfo tWgInfo = topicInfo.getWorkloadGroupInfo();
if (tWgInfo.isSetTag() && !Backend.isMatchWorkloadGroupTag(
tWgInfo.getTag(), beTagSet)) {
if (tWgInfo.isSetTag() && !tWgInfo.getTag().equals(beComputeGroup)) {
Comment thread
yiguolei marked this conversation as resolved.
Resolved
// currently TopicInfo could not contain both policy and workload group,
// so we can remove TopicInfo directly.
topicIter.remove();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,13 @@ public long createLoadJobFromStmt(LoadStmt stmt) throws DdlException, UserExcept
}

if (Config.enable_workload_group) {
loadJob.settWorkloadGroups(
Env.getCurrentEnv().getWorkloadGroupMgr().getWorkloadGroup(ConnectContext.get()));
try {
loadJob.settWorkloadGroups(
Env.getCurrentEnv().getWorkloadGroupMgr().getWorkloadGroup(ConnectContext.get()));

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

我们这为啥都是list

} catch (Throwable t) {
LOG.info("Get workload group failed when create load job,", t);
throw t;
}
}

Env.getCurrentEnv().getEditLog().logCreateLoadJob(loadJob);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,12 @@
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.cloud.system.CloudSystemInfoService;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.planner.StreamLoadPlanner;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.thrift.TKafkaLoadInfo;
import org.apache.doris.thrift.TLoadSourceType;
Expand All @@ -35,7 +37,10 @@
import org.apache.doris.thrift.TUniqueId;

import com.google.common.base.Joiner;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -45,6 +50,8 @@
public class KafkaTaskInfo extends RoutineLoadTaskInfo {
private RoutineLoadManager routineLoadManager = Env.getCurrentEnv().getRoutineLoadManager();

private static final Logger LOG = LogManager.getLogger(KafkaTaskInfo.class);

// <partitionId, offset to be consumed>
private Map<Integer, Long> partitionIdToOffset;

Expand Down Expand Up @@ -138,21 +145,39 @@ private TPipelineFragmentParams rePlan(RoutineLoadJob routineLoadJob) throws Use
tPlanFragment.getOutputSink().getOlapTableSink().setTxnId(txnId);

if (Config.enable_workload_group) {
long wgId = routineLoadJob.getWorkloadId();
List<TPipelineWorkloadGroup> tWgList = new ArrayList<>();
if (wgId > 0) {
tWgList = Env.getCurrentEnv().getWorkloadGroupMgr()
.getTWorkloadGroupById(wgId);
try {
long wgId = routineLoadJob.getWorkloadId();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

为啥叫 getWorkloadId?

List<TPipelineWorkloadGroup> tWgList = new ArrayList<>();
if (wgId > 0) {
tWgList = Env.getCurrentEnv().getWorkloadGroupMgr()
.getTWorkloadGroupById(wgId);
}

if (tWgList.size() == 0) {
throw new UserException("can not find workload group, id=" + wgId);
ConnectContext tmpContext = new ConnectContext();
if (Config.isCloudMode()) {
String clusterName = ((CloudSystemInfoService) Env.getCurrentSystemInfo())
.getClusterNameByClusterId(routineLoadJob.getCloudClusterId());
if (Strings.isNullOrEmpty(clusterName)) {
String err = String.format("cluster name is empty, cluster id is %s",
routineLoadJob.getCloudClusterId());
LOG.warn(err);
throw new UserException(err);
}
tmpContext.setCloudCluster(clusterName);
}
tmpContext.setCurrentUserIdentity(routineLoadJob.getUserIdentity());
tmpContext.setQualifiedUser(routineLoadJob.getUserIdentity().getQualifiedUser());
tWgList = Env.getCurrentEnv().getWorkloadGroupMgr().getWorkloadGroup(tmpContext);
}
} else {
tWgList = Env.getCurrentEnv().getWorkloadGroupMgr()
.getWorkloadGroupByUser(routineLoadJob.getUserIdentity(), false);
}
if (tWgList.size() != 0) {
tExecPlanFragmentParams.setWorkloadGroups(tWgList);
if (tWgList.size() != 0) {
tExecPlanFragmentParams.setWorkloadGroups(tWgList);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

为啥是list?

}
} catch (Throwable t) {
LOG.info("Get workload group failed when replan kafka, job id:{} , ", routineLoadJob.getTxnId(), t);
throw t;
}

}

return tExecPlanFragmentParams;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5301,9 +5301,10 @@ public LogicalPlan visitShowCreateMaterializedView(ShowCreateMaterializedViewCon

@Override
public LogicalPlan visitAlterWorkloadGroup(AlterWorkloadGroupContext ctx) {
String cgName = ctx.computeGroup == null ? "" : stripQuotes(ctx.computeGroup.getText());
Map<String, String> properties = ctx.propertyClause() != null
? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
return new AlterWorkloadGroupCommand(ctx.name.getText(), properties);
return new AlterWorkloadGroupCommand(cgName, ctx.name.getText(), properties);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

不要传空。 传递default

}

@Override
Expand Down Expand Up @@ -5585,10 +5586,11 @@ public LogicalPlan visitDropEncryptkey(DropEncryptkeyContext ctx) {
@Override
public LogicalPlan visitCreateWorkloadGroup(CreateWorkloadGroupContext ctx) {
String workloadGroupName = stripQuotes(ctx.name.getText());
String cgName = ctx.computeGroup == null ? "" : stripQuotes(ctx.computeGroup.getText());
boolean ifNotExists = ctx.EXISTS() != null;
Map<String, String> properties = ctx.propertyClause() != null
? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
return new CreateWorkloadGroupCommand(workloadGroupName, ifNotExists, properties);
return new CreateWorkloadGroupCommand(cgName, workloadGroupName, ifNotExists, properties);
}

@Override
Expand Down Expand Up @@ -5630,7 +5632,8 @@ public LogicalPlan visitDropUser(DropUserContext ctx) {

@Override
public LogicalPlan visitDropWorkloadGroup(DropWorkloadGroupContext ctx) {
return new DropWorkloadGroupCommand(ctx.name.getText(), ctx.EXISTS() != null);
String cgName = ctx.computeGroup == null ? "" : stripQuotes(ctx.computeGroup.getText());
return new DropWorkloadGroupCommand(cgName, ctx.name.getText(), ctx.EXISTS() != null);
}

@Override
Expand Down
Loading