Skip to content

Commit 6b7560b

Browse files
darionyaphetdarionyaphet
authored
Support Cluster Management (#3343)
* Replace group when create space * Support white list * fix test case * fix critical27 comment * fix python test case
1 parent 58914a7 commit 6b7560b

File tree

91 files changed

+3787
-1921
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

91 files changed

+3787
-1921
lines changed

‎.linters/cpp/checkKeyword.py

+4
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,10 @@
111111
'KW_EXPLAIN',
112112
'KW_UNWIND',
113113
'KW_CASE',
114+
'KW_HOSTS',
115+
'KW_ZONE',
116+
'KW_ZONES',
117+
'KW_RENAME',
114118
]
115119

116120

‎src/clients/meta/MetaClient.cpp

+67-17
Original file line numberDiff line numberDiff line change
@@ -780,8 +780,8 @@ Status MetaClient::handleResponse(const RESP& resp) {
780780
return Status::Error("Part not existed!");
781781
case nebula::cpp2::ErrorCode::E_USER_NOT_FOUND:
782782
return Status::Error("User not existed!");
783-
case nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND:
784-
return Status::Error("Group not existed!");
783+
case nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND:
784+
return Status::Error("Machine not existed!");
785785
case nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND:
786786
return Status::Error("Zone not existed!");
787787
case nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND:
@@ -3036,17 +3036,65 @@ void MetaClient::loadLeader(const std::vector<cpp2::HostItem>& hostItems,
30363036
}
30373037
}
30383038

3039-
folly::Future<StatusOr<bool>> MetaClient::addZone(std::string zoneName,
3040-
std::vector<HostAddr> nodes) {
3041-
cpp2::AddZoneReq req;
3042-
req.set_zone_name(std::move(zoneName));
3043-
req.set_nodes(std::move(nodes));
3039+
folly::Future<StatusOr<bool>> MetaClient::addHosts(std::vector<HostAddr> hosts) {
3040+
cpp2::AddHostsReq req;
3041+
req.set_hosts(std::move(hosts));
3042+
3043+
folly::Promise<StatusOr<bool>> promise;
3044+
auto future = promise.getFuture();
3045+
getResponse(
3046+
std::move(req),
3047+
[](auto client, auto request) { return client->future_addHosts(request); },
3048+
[](cpp2::ExecResp&& resp) -> bool {
3049+
return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
3050+
},
3051+
std::move(promise));
3052+
return future;
3053+
}
3054+
3055+
folly::Future<StatusOr<bool>> MetaClient::dropHosts(std::vector<HostAddr> hosts) {
3056+
cpp2::DropHostsReq req;
3057+
req.set_hosts(std::move(hosts));
3058+
3059+
folly::Promise<StatusOr<bool>> promise;
3060+
auto future = promise.getFuture();
3061+
getResponse(
3062+
std::move(req),
3063+
[](auto client, auto request) { return client->future_dropHosts(request); },
3064+
[](cpp2::ExecResp&& resp) -> bool {
3065+
return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
3066+
},
3067+
std::move(promise));
3068+
return future;
3069+
}
3070+
3071+
folly::Future<StatusOr<bool>> MetaClient::mergeZone(std::vector<std::string> zones,
3072+
std::string zoneName) {
3073+
cpp2::MergeZoneReq req;
3074+
req.set_zone_name(zoneName);
3075+
req.set_zones(zones);
3076+
folly::Promise<StatusOr<bool>> promise;
3077+
auto future = promise.getFuture();
3078+
getResponse(
3079+
std::move(req),
3080+
[](auto client, auto request) { return client->future_mergeZone(request); },
3081+
[](cpp2::ExecResp&& resp) -> bool {
3082+
return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
3083+
},
3084+
std::move(promise));
3085+
return future;
3086+
}
30443087

3088+
folly::Future<StatusOr<bool>> MetaClient::renameZone(std::string originalZoneName,
3089+
std::string zoneName) {
3090+
cpp2::RenameZoneReq req;
3091+
req.set_original_zone_name(originalZoneName);
3092+
req.set_zone_name(zoneName);
30453093
folly::Promise<StatusOr<bool>> promise;
30463094
auto future = promise.getFuture();
30473095
getResponse(
30483096
std::move(req),
3049-
[](auto client, auto request) { return client->future_addZone(request); },
3097+
[](auto client, auto request) { return client->future_renameZone(request); },
30503098
[](cpp2::ExecResp&& resp) -> bool {
30513099
return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
30523100
},
@@ -3070,33 +3118,35 @@ folly::Future<StatusOr<bool>> MetaClient::dropZone(std::string zoneName) {
30703118
return future;
30713119
}
30723120

3073-
folly::Future<StatusOr<bool>> MetaClient::addHostIntoZone(HostAddr node, std::string zoneName) {
3074-
cpp2::AddHostIntoZoneReq req;
3075-
req.set_node(node);
3121+
folly::Future<StatusOr<bool>> MetaClient::splitZone(
3122+
std::string zoneName, std::unordered_map<std::string, std::vector<HostAddr>>) {
3123+
cpp2::SplitZoneReq req;
30763124
req.set_zone_name(zoneName);
3077-
30783125
folly::Promise<StatusOr<bool>> promise;
30793126
auto future = promise.getFuture();
30803127
getResponse(
30813128
std::move(req),
3082-
[](auto client, auto request) { return client->future_addHostIntoZone(request); },
3129+
[](auto client, auto request) { return client->future_splitZone(request); },
30833130
[](cpp2::ExecResp&& resp) -> bool {
30843131
return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
30853132
},
30863133
std::move(promise));
30873134
return future;
30883135
}
30893136

3090-
folly::Future<StatusOr<bool>> MetaClient::dropHostFromZone(HostAddr node, std::string zoneName) {
3091-
cpp2::DropHostFromZoneReq req;
3092-
req.set_node(node);
3137+
folly::Future<StatusOr<bool>> MetaClient::addHostsIntoZone(std::vector<HostAddr> hosts,
3138+
std::string zoneName,
3139+
bool isNew) {
3140+
cpp2::AddHostsIntoZoneReq req;
3141+
req.set_hosts(hosts);
30933142
req.set_zone_name(zoneName);
3143+
req.set_is_new(isNew);
30943144

30953145
folly::Promise<StatusOr<bool>> promise;
30963146
auto future = promise.getFuture();
30973147
getResponse(
30983148
std::move(req),
3099-
[](auto client, auto request) { return client->future_dropHostFromZone(request); },
3149+
[](auto client, auto request) { return client->future_addHostsIntoZone(request); },
31003150
[](cpp2::ExecResp&& resp) -> bool {
31013151
return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED;
31023152
},

‎src/clients/meta/MetaClient.h

+12-3
Original file line numberDiff line numberDiff line change
@@ -597,13 +597,22 @@ class MetaClient {
597597

598598
StatusOr<LeaderInfo> getLeaderInfo();
599599

600-
folly::Future<StatusOr<bool>> addZone(std::string zoneName, std::vector<HostAddr> nodes);
600+
folly::Future<StatusOr<bool>> addHosts(std::vector<HostAddr> hosts);
601+
602+
folly::Future<StatusOr<bool>> dropHosts(std::vector<HostAddr> hosts);
603+
604+
folly::Future<StatusOr<bool>> mergeZone(std::vector<std::string> zones, std::string zoneName);
605+
606+
folly::Future<StatusOr<bool>> renameZone(std::string originalZoneName, std::string zoneName);
601607

602608
folly::Future<StatusOr<bool>> dropZone(std::string zoneName);
603609

604-
folly::Future<StatusOr<bool>> addHostIntoZone(HostAddr node, std::string zoneName);
610+
folly::Future<StatusOr<bool>> splitZone(
611+
std::string zoneName, std::unordered_map<std::string, std::vector<HostAddr>> zones);
605612

606-
folly::Future<StatusOr<bool>> dropHostFromZone(HostAddr node, std::string zoneName);
613+
folly::Future<StatusOr<bool>> addHostsIntoZone(std::vector<HostAddr> hosts,
614+
std::string zoneName,
615+
bool isNew);
607616

608617
folly::Future<StatusOr<std::vector<HostAddr>>> getZone(std::string zoneName);
609618

‎src/common/graph/Response.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
X(E_TAG_PROP_NOT_FOUND, -10) \
3636
X(E_ROLE_NOT_FOUND, -11) \
3737
X(E_CONFIG_NOT_FOUND, -12) \
38-
X(E_GROUP_NOT_FOUND, -13) \
38+
X(E_MACHINE_NOT_FOUND, -13) \
3939
X(E_ZONE_NOT_FOUND, -14) \
4040
X(E_LISTENER_NOT_FOUND, -15) \
4141
X(E_PART_NOT_FOUND, -16) \

‎src/common/utils/MetaKeyUtils.cpp

+18-24
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ static const std::unordered_map<std::string, std::pair<std::string, bool>> syste
2121
{"users", {"__users__", true}},
2222
{"hosts", {"__hosts__", false}},
2323
{"versions", {"__versions__", false}},
24+
{"machines", {"__machines__", false}},
2425
{"snapshots", {"__snapshots__", false}},
2526
{"configs", {"__configs__", true}},
2627
{"groups", {"__groups__", true}},
@@ -58,8 +59,9 @@ static const std::unordered_map<
5859
// clang-format off
5960
static const std::string kSpacesTable = tableMaps.at("spaces").first; // NOLINT
6061
static const std::string kPartsTable = tableMaps.at("parts").first; // NOLINT
61-
static const std::string kHostsTable = systemTableMaps.at("hosts").first; // NOLINT
6262
static const std::string kVersionsTable = systemTableMaps.at("versions").first; // NOLINT
63+
static const std::string kHostsTable = systemTableMaps.at("hosts").first; // NOLINT
64+
static const std::string kMachinesTable = systemTableMaps.at("machines").first; // NOLINT
6365
static const std::string kTagsTable = tableMaps.at("tags").first; // NOLINT
6466
static const std::string kEdgesTable = tableMaps.at("edges").first; // NOLINT
6567
static const std::string kIndexesTable = tableMaps.at("indexes").first; // NOLINT
@@ -239,6 +241,21 @@ std::vector<HostAddr> MetaKeyUtils::parsePartValV2(folly::StringPiece val) {
239241
return ret;
240242
}
241243

244+
std::string MetaKeyUtils::machineKey(std::string addr, Port port) {
245+
std::string key;
246+
HostAddr h(addr, port);
247+
key.append(kMachinesTable.data(), kMachinesTable.size())
248+
.append(MetaKeyUtils::serializeHostAddr(h));
249+
return key;
250+
}
251+
252+
const std::string& MetaKeyUtils::machinePrefix() { return kMachinesTable; }
253+
254+
HostAddr MetaKeyUtils::parseMachineKey(folly::StringPiece key) {
255+
key.advance(kMachinesTable.size());
256+
return MetaKeyUtils::deserializeHostAddr(key);
257+
}
258+
242259
std::string MetaKeyUtils::hostKey(std::string addr, Port port) { return hostKeyV2(addr, port); }
243260

244261
std::string MetaKeyUtils::hostKeyV2(std::string addr, Port port) {
@@ -961,29 +978,6 @@ MetaKeyUtils::parseBalanceTaskVal(const folly::StringPiece& rawVal) {
961978
return std::make_tuple(status, ret, start, end);
962979
}
963980

964-
std::string MetaKeyUtils::groupKey(const std::string& group) {
965-
std::string key;
966-
key.reserve(kGroupsTable.size() + group.size());
967-
key.append(kGroupsTable.data(), kGroupsTable.size()).append(group);
968-
return key;
969-
}
970-
971-
std::string MetaKeyUtils::groupVal(const std::vector<std::string>& zones) {
972-
return folly::join(",", zones);
973-
}
974-
975-
const std::string& MetaKeyUtils::groupPrefix() { return kGroupsTable; }
976-
977-
std::string MetaKeyUtils::parseGroupName(folly::StringPiece rawData) {
978-
return rawData.subpiece(kGroupsTable.size(), rawData.size()).toString();
979-
}
980-
981-
std::vector<std::string> MetaKeyUtils::parseZoneNames(folly::StringPiece rawData) {
982-
std::vector<std::string> zones;
983-
folly::split(',', rawData.str(), zones);
984-
return zones;
985-
}
986-
987981
std::string MetaKeyUtils::zoneKey(const std::string& zone) {
988982
std::string key;
989983
key.reserve(kZonesTable.size() + zone.size());

‎src/common/utils/MetaKeyUtils.h

+6-10
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,12 @@ class MetaKeyUtils final {
110110

111111
static std::vector<HostAddr> parsePartValV2(folly::StringPiece val);
112112

113+
static std::string machineKey(std::string ip, Port port);
114+
115+
static const std::string& machinePrefix();
116+
117+
static HostAddr parseMachineKey(folly::StringPiece key);
118+
113119
static std::string hostKey(std::string ip, Port port);
114120

115121
static std::string hostKeyV2(std::string addr, Port port);
@@ -286,16 +292,6 @@ class MetaKeyUtils final {
286292
static std::tuple<BalanceTaskStatus, BalanceTaskResult, int64_t, int64_t> parseBalanceTaskVal(
287293
const folly::StringPiece& rawVal);
288294

289-
static std::string groupKey(const std::string& group);
290-
291-
static std::string groupVal(const std::vector<std::string>& zones);
292-
293-
static const std::string& groupPrefix();
294-
295-
static std::string parseGroupName(folly::StringPiece rawData);
296-
297-
static std::vector<std::string> parseZoneNames(folly::StringPiece rawData);
298-
299295
static std::string zoneKey(const std::string& zone);
300296

301297
static std::string zoneVal(const std::vector<HostAddr>& hosts);

‎src/common/utils/test/MetaKeyUtilsTest.cpp

+29-11
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,28 @@ TEST(MetaKeyUtilsTest, SpaceKeyTest) {
2424
spaceDesc.set_replica_factor(3);
2525
auto spaceVal = MetaKeyUtils::spaceVal(spaceDesc);
2626
ASSERT_EQ("default", MetaKeyUtils::spaceName(spaceVal));
27-
ASSERT_EQ(100, MetaKeyUtils::parseSpace(spaceVal).get_partition_num());
28-
ASSERT_EQ(3, MetaKeyUtils::parseSpace(spaceVal).get_replica_factor());
27+
auto properties = MetaKeyUtils::parseSpace(spaceVal);
28+
ASSERT_EQ(100, properties.get_partition_num());
29+
ASSERT_EQ(3, properties.get_replica_factor());
30+
}
31+
32+
TEST(MetaKeyUtilsTest, SpaceKeyWithZonesTest) {
33+
auto prefix = MetaKeyUtils::spacePrefix();
34+
ASSERT_EQ("__spaces__", prefix);
35+
auto spaceKey = MetaKeyUtils::spaceKey(101);
36+
ASSERT_EQ(101, MetaKeyUtils::spaceId(spaceKey));
37+
meta::cpp2::SpaceDesc spaceDesc;
38+
spaceDesc.set_space_name("default");
39+
spaceDesc.set_partition_num(100);
40+
spaceDesc.set_replica_factor(3);
41+
std::vector<std::string> zoneNames{"z1", "z2", "z3"};
42+
spaceDesc.set_zone_names(std::move(zoneNames));
43+
auto spaceVal = MetaKeyUtils::spaceVal(spaceDesc);
44+
ASSERT_EQ("default", MetaKeyUtils::spaceName(spaceVal));
45+
auto properties = MetaKeyUtils::parseSpace(spaceVal);
46+
ASSERT_EQ(100, properties.get_partition_num());
47+
ASSERT_EQ(3, properties.get_replica_factor());
48+
ASSERT_EQ(3, properties.get_zone_names().size());
2949
}
3050

3151
TEST(MetaKeyUtilsTest, PartKeyTest) {
@@ -75,6 +95,13 @@ TEST(MetaKeyUtilsTest, storeStrIpCodecTest) {
7595
}
7696
}
7797

98+
{
99+
// kMachinesTable : key
100+
auto key = MetaKeyUtils::machineKey(hostnames[0], ports[0]);
101+
auto host = MetaKeyUtils::parseMachineKey(key);
102+
ASSERT_EQ(host.host, hostnames[0]);
103+
ASSERT_EQ(host.port, ports[0]);
104+
}
78105
{
79106
// kHostsTable : key
80107
auto key = MetaKeyUtils::hostKey(hostnames[0], ports[0]);
@@ -183,15 +210,6 @@ TEST(MetaKeyUtilsTest, TagTest) {
183210
ASSERT_EQ(parsedSchema, schema);
184211
}
185212

186-
TEST(MetaKeyUtilsTest, GroupTest) {
187-
auto groupKey = MetaKeyUtils::groupKey("test_group");
188-
ASSERT_EQ("test_group", MetaKeyUtils::parseGroupName(groupKey));
189-
190-
std::vector<std::string> zones = {"zone_0", "zone_1", "zone_2"};
191-
auto groupValue = MetaKeyUtils::groupVal(zones);
192-
ASSERT_EQ(zones, MetaKeyUtils::parseZoneNames(groupValue));
193-
}
194-
195213
TEST(MetaKeyUtilsTest, ZoneTest) {
196214
auto zoneKey = MetaKeyUtils::zoneKey("test_zone");
197215
ASSERT_EQ("test_zone", MetaKeyUtils::parseZoneName(zoneKey));

‎src/graph/executor/CMakeLists.txt

+2
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ nebula_add_library(
4444
algo/ProduceAllPathsExecutor.cpp
4545
algo/CartesianProductExecutor.cpp
4646
algo/SubgraphExecutor.cpp
47+
admin/AddHostsExecutor.cpp
48+
admin/DropHostsExecutor.cpp
4749
admin/SwitchSpaceExecutor.cpp
4850
admin/CreateUserExecutor.cpp
4951
admin/DropUserExecutor.cpp

‎src/graph/executor/Executor.cpp

+18-7
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,14 @@
1616
#include "graph/context/ExecutionContext.h"
1717
#include "graph/context/QueryContext.h"
1818
#include "graph/executor/ExecutionError.h"
19+
#include "graph/executor/admin/AddHostsExecutor.h"
1920
#include "graph/executor/admin/ChangePasswordExecutor.h"
2021
#include "graph/executor/admin/CharsetExecutor.h"
2122
#include "graph/executor/admin/ConfigExecutor.h"
2223
#include "graph/executor/admin/CreateUserExecutor.h"
2324
#include "graph/executor/admin/DescribeUserExecutor.h"
2425
#include "graph/executor/admin/DownloadExecutor.h"
26+
#include "graph/executor/admin/DropHostsExecutor.h"
2527
#include "graph/executor/admin/DropUserExecutor.h"
2628
#include "graph/executor/admin/GrantRoleExecutor.h"
2729
#include "graph/executor/admin/IngestExecutor.h"
@@ -442,20 +444,29 @@ Executor *Executor::makeExecutor(QueryContext *qctx, const PlanNode *node) {
442444
case PlanNode::Kind::kSubgraph: {
443445
return pool->add(new SubgraphExecutor(node, qctx));
444446
}
445-
case PlanNode::Kind::kAddZone: {
446-
return pool->add(new AddZoneExecutor(node, qctx));
447+
case PlanNode::Kind::kAddHosts: {
448+
return pool->add(new AddHostsExecutor(node, qctx));
449+
}
450+
case PlanNode::Kind::kDropHosts: {
451+
return pool->add(new DropHostsExecutor(node, qctx));
452+
}
453+
case PlanNode::Kind::kMergeZone: {
454+
return pool->add(new MergeZoneExecutor(node, qctx));
455+
}
456+
case PlanNode::Kind::kRenameZone: {
457+
return pool->add(new RenameZoneExecutor(node, qctx));
447458
}
448459
case PlanNode::Kind::kDropZone: {
449460
return pool->add(new DropZoneExecutor(node, qctx));
450461
}
462+
case PlanNode::Kind::kSplitZone: {
463+
return pool->add(new SplitZoneExecutor(node, qctx));
464+
}
451465
case PlanNode::Kind::kDescribeZone: {
452466
return pool->add(new DescribeZoneExecutor(node, qctx));
453467
}
454-
case PlanNode::Kind::kAddHostIntoZone: {
455-
return pool->add(new AddHostIntoZoneExecutor(node, qctx));
456-
}
457-
case PlanNode::Kind::kDropHostFromZone: {
458-
return pool->add(new DropHostFromZoneExecutor(node, qctx));
468+
case PlanNode::Kind::kAddHostsIntoZone: {
469+
return pool->add(new AddHostsIntoZoneExecutor(node, qctx));
459470
}
460471
case PlanNode::Kind::kShowZones: {
461472
return pool->add(new ListZonesExecutor(node, qctx));

0 commit comments

Comments
 (0)