Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion cloud/src/meta-service/injection_point_http.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,17 @@ HttpResponse set_sleep(const std::string& point, const brpc::URI& uri) {
return http_json_reply(MetaServiceCode::OK, "OK");
}

HttpResponse set_wait(const std::string& point) {
auto* sp = SyncPoint::get_instance();
sp->set_call_back(point, [point, sp](auto&&) {
LOG(INFO) << "injection point waiting, point=" << point;
while (sp->get_enable() && sp->has_point(point)) {
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
});
return http_json_reply(MetaServiceCode::OK, "OK");
}

HttpResponse set_return(const std::string& point, const brpc::URI& uri) {
auto sp = SyncPoint::get_instance();
sp->set_call_back(point, [point](auto&& args) {
Expand All @@ -293,7 +304,9 @@ HttpResponse handle_set(const brpc::URI& uri) {
if (behavior.empty()) {
return http_json_reply(MetaServiceCode::INVALID_ARGUMENT, "empty behavior");
}
if (behavior == "sleep") {
if (behavior == "wait") {
return set_wait(point);
} else if (behavior == "sleep") {
return set_sleep(point, uri);
} else if (behavior == "return") {
return set_return(point, uri);
Expand Down Expand Up @@ -349,6 +362,8 @@ HttpResponse handle_disable(const brpc::URI& uri) {
// curl "ms_ip:port/MetaService/http/v1/injection_point?token=greedisgood9999&op=disable"
// ```

// behavior=wait blocks at a point until it is cleared or injection is disabled.
//
// clear all injection points
// ```
// curl "ms_ip:port/MetaService/http/v1/injection_point?token=greedisgood9999&op=clear"
Expand Down
1 change: 1 addition & 0 deletions cloud/src/meta-service/meta_service.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2675,6 +2675,7 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle
msg = "no rowset meta";
return;
}
TEST_INJECTION_POINT("commit_rowset::before_get_instance_id:" + request->cloud_unique_id());
instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id());
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
Expand Down
188 changes: 76 additions & 112 deletions cloud/src/meta-service/meta_service_resource.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3263,133 +3263,95 @@ void handle_drop_node(const std::string& instance_id, const AlterClusterRequest*
msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
}

void handle_decommission_node(const std::string& instance_id, const AlterClusterRequest* request,
std::shared_ptr<ResourceManager> resource_mgr, std::string& msg,
MetaServiceCode& code) {
resource_mgr->check_cluster_params_valid(request->cluster(), &msg, false, false);
if (msg != "") {
LOG(WARNING) << msg;
return;
static bool matches_node(const NodeInfoPB& node, const NodeInfoPB& requested) {
if (node.cloud_unique_id() != requested.cloud_unique_id() ||
node.heartbeat_port() != requested.heartbeat_port()) {
return false;
}
if (node.has_ip() && requested.has_ip()) {
return node.ip() == requested.ip();
}
return node.has_host() && requested.has_host() && node.host() == requested.host();
}

std::string be_unique_id = (request->cluster().nodes())[0].cloud_unique_id();
static std::string resolve_cluster_from_node(ResourceManager& resource_mgr,
const NodeInfoPB& requested, ClusterPB& cluster) {
std::vector<NodeInfo> nodes;
std::string err = resource_mgr->get_node(be_unique_id, &nodes);
if (!err.empty()) {
LOG(INFO) << "failed to check instance info, err=" << err;
msg = err;
return;
}

std::vector<NodeInfo> decomission_nodes;
for (auto& node : nodes) {
for (auto req_node : request->cluster().nodes()) {
bool ip_processed = false;
if (node.node_info.has_ip() && req_node.has_ip()) {
std::string endpoint =
node.node_info.ip() + ":" + std::to_string(node.node_info.heartbeat_port());
std::string req_endpoint =
req_node.ip() + ":" + std::to_string(req_node.heartbeat_port());
if (endpoint == req_endpoint) {
decomission_nodes.push_back(node);
node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
}
ip_processed = true;
}

if (!ip_processed && node.node_info.has_host() && req_node.has_host()) {
std::string endpoint = node.node_info.host() + ":" +
std::to_string(node.node_info.heartbeat_port());
std::string req_endpoint =
req_node.host() + ":" + std::to_string(req_node.heartbeat_port());
if (endpoint == req_endpoint) {
decomission_nodes.push_back(node);
node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
}
}
auto msg = resource_mgr.get_node(requested.cloud_unique_id(), &nodes);
if (!msg.empty()) {
return msg;
}
for (const auto& node : nodes) {
if (matches_node(node.node_info, requested)) {
cluster.set_cluster_id(node.cluster_id);
cluster.set_cluster_name(node.cluster_name);
return "";
}
}
return "failed to find node to update: " + requested.ShortDebugString();
}

{
std::vector<NodeInfo> to_add;
std::vector<NodeInfo>& to_del = decomission_nodes;
msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
}
{
std::vector<NodeInfo>& to_add = decomission_nodes;
std::vector<NodeInfo> to_del;
for (auto& node : to_add) {
node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
LOG(INFO) << "decomission node, "
<< "size: " << to_add.size() << " " << node.node_info.DebugString() << " "
<< node.cluster_id << " " << node.cluster_name;
static std::string update_cluster_node_statuses(ClusterPB& cluster,
const ClusterPB& requested_cluster,
NodeStatusPB status) {
const auto mtime = duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
auto* nodes = cluster.mutable_nodes();
for (const auto& requested : requested_cluster.nodes()) {
NodeInfoPB* matched = nullptr;
for (auto& node : *nodes) {
if (matches_node(node, requested)) {
matched = &node;
break;
}
}
if (matched == nullptr) {
return "failed to find node to update: " + requested.ShortDebugString();
}
msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
matched->set_status(status);
matched->set_mtime(mtime);
}
return "";
}

void handle_notify_decommissioned(const std::string& instance_id,
const AlterClusterRequest* request,
std::shared_ptr<ResourceManager> resource_mgr, std::string& msg,
MetaServiceCode& code) {
resource_mgr->check_cluster_params_valid(request->cluster(), &msg, false, false);
if (msg != "") {
void handle_update_node_status(const std::string& instance_id, const AlterClusterRequest* request,
std::shared_ptr<ResourceManager> resource_mgr, NodeStatusPB status,
std::string& msg) {
const auto& requested_cluster = request->cluster();
resource_mgr->check_cluster_params_valid(requested_cluster, &msg, false, false);
if (!msg.empty()) {
LOG(WARNING) << msg;
return;
}

std::string be_unique_id = (request->cluster().nodes())[0].cloud_unique_id();
std::vector<NodeInfo> nodes;
std::string err = resource_mgr->get_node(be_unique_id, &nodes);
if (!err.empty()) {
LOG(INFO) << "failed to check instance info, err=" << err;
msg = err;
return;
}

std::vector<NodeInfo> decomission_nodes;
for (auto& node : nodes) {
for (auto req_node : request->cluster().nodes()) {
bool ip_processed = false;
if (node.node_info.has_ip() && req_node.has_ip()) {
std::string endpoint =
node.node_info.ip() + ":" + std::to_string(node.node_info.heartbeat_port());
std::string req_endpoint =
req_node.ip() + ":" + std::to_string(req_node.heartbeat_port());
if (endpoint == req_endpoint) {
decomission_nodes.push_back(node);
}
ip_processed = true;
}

if (!ip_processed && node.node_info.has_host() && req_node.has_host()) {
std::string endpoint = node.node_info.host() + ":" +
std::to_string(node.node_info.heartbeat_port());
std::string req_endpoint =
req_node.host() + ":" + std::to_string(req_node.heartbeat_port());
if (endpoint == req_endpoint) {
decomission_nodes.push_back(node);
}
}
}
if (requested_cluster.nodes().empty()) {
msg = "no nodes to update";
return;
}

{
std::vector<NodeInfo> to_add;
std::vector<NodeInfo>& to_del = decomission_nodes;
msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
}
{
std::vector<NodeInfo>& to_add = decomission_nodes;
std::vector<NodeInfo> to_del;
for (auto& node : to_add) {
node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONED);
LOG(INFO) << "notify node decomissioned, "
<< " size: " << to_add.size() << " " << node.node_info.DebugString() << " "
<< node.cluster_id << " " << node.cluster_name;
ClusterInfo cluster;
cluster.cluster.CopyFrom(requested_cluster);
if (cluster.cluster.cluster_id().empty()) {
// Older HTTP callers can identify the cluster through the node alone.
msg = resolve_cluster_from_node(*resource_mgr, requested_cluster.nodes(0), cluster.cluster);
if (!msg.empty()) {
return;
}
msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
}

const auto& cluster_id = cluster.cluster.cluster_id();
const auto& cluster_name = cluster.cluster.cluster_name();
// HTTP-created clusters without a stored ID are distinguished by name.
auto match_cluster = [&cluster_id, &cluster_name](const ClusterPB& candidate) {
return candidate.cluster_id() == cluster_id &&
(!cluster_id.empty() || candidate.cluster_name() == cluster_name);
};
auto update_nodes = [&requested_cluster, status](ClusterPB& current, std::vector<ClusterPB>&) {
return update_cluster_node_statuses(current, requested_cluster, status);
};

// Publish only the final state: removing and re-adding nodes breaks instance lookup.
msg = resource_mgr->update_cluster(instance_id, cluster, match_cluster, update_nodes,
/*replace_if_existing_empty_target_cluster=*/false,
/*allow_cluster_name_fallback=*/true);
}

void handle_rename_cluster(const std::string& instance_id, const ClusterInfo& cluster,
Expand Down Expand Up @@ -3687,10 +3649,12 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller,
handle_drop_node(instance_id, request, resource_mgr(), msg, code);
break;
case AlterClusterRequest::DECOMMISSION_NODE:
handle_decommission_node(instance_id, request, resource_mgr(), msg, code);
handle_update_node_status(instance_id, request, resource_mgr(),
NodeStatusPB::NODE_STATUS_DECOMMISSIONING, msg);
break;
case AlterClusterRequest::NOTIFY_DECOMMISSIONED:
handle_notify_decommissioned(instance_id, request, resource_mgr(), msg, code);
handle_update_node_status(instance_id, request, resource_mgr(),
NodeStatusPB::NODE_STATUS_DECOMMISSIONED, msg);
break;
case AlterClusterRequest::RENAME_CLUSTER: {
// SQL mode, cluster cluster name eq empty cluster name, need drop empty cluster first.
Expand Down
7 changes: 5 additions & 2 deletions cloud/src/resource-manager/resource_manager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -781,14 +781,15 @@ std::string ResourceManager::update_cluster(
const std::string& instance_id, const ClusterInfo& cluster,
std::function<bool(const ClusterPB&)> filter,
std::function<std::string(ClusterPB&, std::vector<ClusterPB>& clusters_in_instance)> action,
bool replace_if_existing_empty_target_cluster) {
bool replace_if_existing_empty_target_cluster, bool allow_cluster_name_fallback) {
std::stringstream ss;
std::string msg;

std::string cluster_id = cluster.cluster.has_cluster_id() ? cluster.cluster.cluster_id() : "";
std::string cluster_name =
cluster.cluster.has_cluster_name() ? cluster.cluster.cluster_name() : "";
if (cluster_id.empty()) {
const bool can_fallback_to_name = allow_cluster_name_fallback && !cluster_name.empty();
if (cluster_id.empty() && !can_fallback_to_name) {
ss << "missing cluster_id=" << cluster_id;
msg = ss.str();
LOG(INFO) << msg;
Expand Down Expand Up @@ -916,6 +917,7 @@ std::string ResourceManager::update_cluster(
<< " instance json=" << proto_to_json(instance);

refresh_instance(instance_id, instance);
TEST_INJECTION_POINT("ResourceManager::node_update:after_refresh");

return msg;
}
Expand Down Expand Up @@ -1383,6 +1385,7 @@ std::string ResourceManager::modify_nodes(const std::string& instance_id,
}

refresh_instance(instance_id, instance);
TEST_INJECTION_POINT("ResourceManager::node_update:after_refresh");

return "";
}
Expand Down
4 changes: 3 additions & 1 deletion cloud/src/resource-manager/resource_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ class ResourceManager {
* @param cluster cluster to update, only cluster name and cluster id are concered
* @param action update operation code snippet
* @param replace_if_existing_empty_target_cluster, find cluster.cluster_name is a empty cluster(no node), drop it
* @param allow_cluster_name_fallback allow an empty ID when the cluster name is set
* @filter filter condition
* @return empty string for success, otherwise failure reason returned
*/
Expand All @@ -97,7 +98,8 @@ class ResourceManager {
std::function<bool(const ClusterPB&)> filter,
std::function<std::string(ClusterPB&, std::vector<ClusterPB>& clusters_in_instance)>
action,
bool replace_if_existing_empty_target_cluster = false);
bool replace_if_existing_empty_target_cluster = false,
bool allow_cluster_name_fallback = false);

/**
* Get instance from underlying storage with given transaction.
Expand Down
Loading