From df4ec5b5d1048c2ef85a0e7a494752d3b4c42a41 Mon Sep 17 00:00:00 2001 From: "darion.yaphet" Date: Tue, 16 Nov 2021 21:12:58 +0800 Subject: [PATCH 1/5] Replace group when create space --- src/common/utils/test/MetaKeyUtilsTest.cpp | 17 ++++ src/graph/executor/admin/SpaceExecutor.cpp | 62 +++++++------- src/graph/validator/AdminValidator.cpp | 8 +- src/interface/meta.thrift | 2 +- src/meta/ActiveHostsMan.cpp | 12 +-- src/meta/ActiveHostsMan.h | 2 +- .../processors/job/BalanceJobExecutor.cpp | 84 +++++++------------ src/meta/processors/job/BalanceJobExecutor.h | 15 ++-- .../processors/parts/CreateSpaceProcessor.cpp | 31 +++---- .../processors/parts/GetSpaceProcessor.cpp | 6 +- .../processors/zone/UpdateZoneProcessor.cpp | 39 +++++---- src/meta/test/BalancerTest.cpp | 61 ++++++++++---- src/meta/test/RestoreProcessorTest.cpp | 20 ----- src/parser/AdminSentences.cpp | 4 +- src/parser/AdminSentences.h | 6 +- src/parser/MaintainSentences.h | 30 ------- src/parser/Sentence.h | 30 +++++++ src/parser/parser.yy | 29 +++++-- 18 files changed, 243 insertions(+), 215 deletions(-) diff --git a/src/common/utils/test/MetaKeyUtilsTest.cpp b/src/common/utils/test/MetaKeyUtilsTest.cpp index a3bc910c80d..791c71ca148 100644 --- a/src/common/utils/test/MetaKeyUtilsTest.cpp +++ b/src/common/utils/test/MetaKeyUtilsTest.cpp @@ -28,6 +28,23 @@ TEST(MetaKeyUtilsTest, SpaceKeyTest) { ASSERT_EQ(3, MetaKeyUtils::parseSpace(spaceVal).get_replica_factor()); } +TEST(MetaKeyUtilsTest, SpaceKeyWithZonesTest) { + auto prefix = MetaKeyUtils::spacePrefix(); + ASSERT_EQ("__spaces__", prefix); + auto spaceKey = MetaKeyUtils::spaceKey(101); + ASSERT_EQ(101, MetaKeyUtils::spaceId(spaceKey)); + meta::cpp2::SpaceDesc spaceDesc; + spaceDesc.set_space_name("default"); + spaceDesc.set_partition_num(100); + spaceDesc.set_replica_factor(3); + std::vector zoneNames{"z1", "z2", "z3"}; + spaceDesc.set_zone_names(std::move(zoneNames)); + auto spaceVal = MetaKeyUtils::spaceVal(spaceDesc); + ASSERT_EQ("default", MetaKeyUtils::spaceName(spaceVal)); + ASSERT_EQ(100, MetaKeyUtils::parseSpace(spaceVal).get_partition_num()); + ASSERT_EQ(3, MetaKeyUtils::parseSpace(spaceVal).get_replica_factor()); +} + TEST(MetaKeyUtilsTest, PartKeyTest) { auto partKey = MetaKeyUtils::partKey(0, 1); auto prefix = MetaKeyUtils::partPrefix(0); diff --git a/src/graph/executor/admin/SpaceExecutor.cpp b/src/graph/executor/admin/SpaceExecutor.cpp index 9b127dab81c..801a97d2a7d 100644 --- a/src/graph/executor/admin/SpaceExecutor.cpp +++ b/src/graph/executor/admin/SpaceExecutor.cpp @@ -80,7 +80,7 @@ folly::Future DescSpaceExecutor::execute() { "Collate", "Vid Type", "Atomic Edge", - "Group", + "Zones", "Comment"}; Row row; row.values.emplace_back(spaceId); @@ -96,11 +96,10 @@ folly::Future DescSpaceExecutor::execute() { sAtomicEdge = true; } row.values.emplace_back(sAtomicEdge); - if (properties.group_name_ref().has_value()) { - row.values.emplace_back(*properties.group_name_ref()); - } else { - row.values.emplace_back("default"); - } + + auto zoneNames = folly::join(",", properties.get_zone_names()); + row.values.emplace_back(zoneNames); + if (properties.comment_ref().has_value()) { row.values.emplace_back(*properties.comment_ref()); } else { @@ -221,36 +220,37 @@ folly::Future ShowCreateSpaceExecutor::execute() { auto fmt = properties.comment_ref().has_value() ? "CREATE SPACE `%s` (partition_num = %d, replica_factor = %d, " "charset = %s, collate = %s, vid_type = %s, atomic_edge = %s" - ") ON %s" + ")%s" " comment = '%s'" : "CREATE SPACE `%s` (partition_num = %d, replica_factor = %d, " "charset = %s, collate = %s, vid_type = %s, atomic_edge = %s" - ") ON %s"; + ")%s"; + auto zoneNames = !properties.get_zone_names().empty() + ? " ON " + folly::join(",", properties.get_zone_names()) + : ""; if (properties.comment_ref().has_value()) { - row.values.emplace_back(folly::stringPrintf( - fmt, - properties.get_space_name().c_str(), - properties.get_partition_num(), - properties.get_replica_factor(), - properties.get_charset_name().c_str(), - properties.get_collate_name().c_str(), - SchemaUtil::typeToString(properties.get_vid_type()).c_str(), - sAtomicEdge.c_str(), - properties.group_name_ref().has_value() ? properties.get_group_name()->c_str() - : "default", - properties.comment_ref()->c_str())); + row.values.emplace_back( + folly::stringPrintf(fmt, + properties.get_space_name().c_str(), + properties.get_partition_num(), + properties.get_replica_factor(), + properties.get_charset_name().c_str(), + properties.get_collate_name().c_str(), + SchemaUtil::typeToString(properties.get_vid_type()).c_str(), + sAtomicEdge.c_str(), + zoneNames.c_str(), + properties.comment_ref()->c_str())); } else { - row.values.emplace_back(folly::stringPrintf( - fmt, - properties.get_space_name().c_str(), - properties.get_partition_num(), - properties.get_replica_factor(), - properties.get_charset_name().c_str(), - properties.get_collate_name().c_str(), - SchemaUtil::typeToString(properties.get_vid_type()).c_str(), - sAtomicEdge.c_str(), - properties.group_name_ref().has_value() ? properties.group_name_ref()->c_str() - : "default")); + row.values.emplace_back( + folly::stringPrintf(fmt, + properties.get_space_name().c_str(), + properties.get_partition_num(), + properties.get_replica_factor(), + properties.get_charset_name().c_str(), + properties.get_collate_name().c_str(), + SchemaUtil::typeToString(properties.get_vid_type()).c_str(), + sAtomicEdge.c_str(), + zoneNames.c_str())); } dataSet.rows.emplace_back(std::move(row)); return finish(ResultBuilder() diff --git a/src/graph/validator/AdminValidator.cpp b/src/graph/validator/AdminValidator.cpp index 278bfb3d344..e71fcdc3892 100644 --- a/src/graph/validator/AdminValidator.cpp +++ b/src/graph/validator/AdminValidator.cpp @@ -20,8 +20,8 @@ Status CreateSpaceValidator::validateImpl() { ifNotExist_ = sentence->isIfNotExist(); auto status = Status::OK(); spaceDesc_.set_space_name(std::move(*(sentence->spaceName()))); - if (sentence->groupName()) { - spaceDesc_.set_group_name(std::move(*(sentence->groupName()))); + if (sentence->zoneNames()) { + spaceDesc_.set_zone_names(sentence->zoneNames()->zoneNames()); } StatusOr retStatusOr; std::string result; @@ -103,10 +103,6 @@ Status CreateSpaceValidator::validateImpl() { spaceDesc_.set_comment(*sentence->comment()); } - if (sentence->groupName() != nullptr && *sentence->groupName() == "default") { - return Status::SemanticError("Group default conflict"); - } - // if charset and collate are not specified, set default value if (!(*spaceDesc_.charset_name_ref()).empty() && !(*spaceDesc_.collate_name_ref()).empty()) { NG_RETURN_IF_ERROR(charsetInfo->charsetAndCollateMatch(*spaceDesc_.charset_name_ref(), diff --git a/src/interface/meta.thrift b/src/interface/meta.thrift index f9bd4e1f92d..1cb2bd43446 100644 --- a/src/interface/meta.thrift +++ b/src/interface/meta.thrift @@ -112,7 +112,7 @@ struct SpaceDesc { 4: binary charset_name, 5: binary collate_name, 6: ColumnTypeDef vid_type = {"type": common.PropertyType.FIXED_STRING, "type_length": 8}, - 7: optional binary group_name, + 7: list zone_names, 8: optional IsolationLevel isolation_level, 9: optional binary comment, } diff --git a/src/meta/ActiveHostsMan.cpp b/src/meta/ActiveHostsMan.cpp index 5534c656f4a..05a435a1a47 100644 --- a/src/meta/ActiveHostsMan.cpp +++ b/src/meta/ActiveHostsMan.cpp @@ -145,7 +145,7 @@ ErrorOr> ActiveHostsMan::getActiv return activeHosts; } -ErrorOr> ActiveHostsMan::getActiveHostsWithGroup( +ErrorOr> ActiveHostsMan::getActiveHostsWithZones( kvstore::KVStore* kv, GraphSpaceID spaceId, int32_t expiredTTL) { std::string spaceValue; std::vector activeHosts; @@ -158,15 +158,7 @@ ErrorOr> ActiveHostsMan::getActiv std::string groupValue; auto space = MetaKeyUtils::parseSpace(std::move(spaceValue)); - auto groupKey = MetaKeyUtils::groupKey(*space.group_name_ref()); - retCode = kv->get(kDefaultSpaceId, kDefaultPartId, groupKey, &groupValue); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Get group " << *space.group_name_ref() - << " failed, error: " << apache::thrift::util::enumNameSafe(retCode); - return retCode; - } - - auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(groupValue)); + auto zoneNames = *space.zone_names_ref(); for (const auto& zoneName : zoneNames) { auto hostsRet = getActiveHostsInZone(kv, zoneName, expiredTTL); if (!nebula::ok(hostsRet)) { diff --git a/src/meta/ActiveHostsMan.h b/src/meta/ActiveHostsMan.h index 7c1e4e9c26f..4bc318559b2 100644 --- a/src/meta/ActiveHostsMan.h +++ b/src/meta/ActiveHostsMan.h @@ -116,7 +116,7 @@ class ActiveHostsMan final { static ErrorOr> getActiveHostsInZone( kvstore::KVStore* kv, const std::string& zoneName, int32_t expiredTTL = 0); - static ErrorOr> getActiveHostsWithGroup( + static ErrorOr> getActiveHostsWithZones( kvstore::KVStore* kv, GraphSpaceID spaceId, int32_t expiredTTL = 0); static ErrorOr> getActiveAdminHosts( diff --git a/src/meta/processors/job/BalanceJobExecutor.cpp b/src/meta/processors/job/BalanceJobExecutor.cpp index 56ebaaaba4e..ae38d659d92 100644 --- a/src/meta/processors/job/BalanceJobExecutor.cpp +++ b/src/meta/processors/job/BalanceJobExecutor.cpp @@ -110,7 +110,7 @@ nebula::cpp2::ErrorCode BalanceJobExecutor::getAllSpaces( while (iter->valid()) { auto spaceId = MetaKeyUtils::spaceId(iter->key()); auto properties = MetaKeyUtils::parseSpace(iter->val()); - bool zoned = properties.group_name_ref().has_value(); + bool zoned = !properties.get_zone_names().empty(); spaces.emplace_back(spaceId, *properties.replica_factor_ref(), zoned); iter->next(); } @@ -146,9 +146,9 @@ nebula::cpp2::ErrorCode DataBalanceJobExecutor::buildBalancePlan() { for (const auto& spaceInfo : spaces) { auto spaceId = std::get<0>(spaceInfo); auto spaceReplica = std::get<1>(spaceInfo); - auto dependentOnGroup = std::get<2>(spaceInfo); + auto dependentOnZone = std::get<2>(spaceInfo); LOG(INFO) << "Balance Space " << spaceId; - auto taskRet = genTasks(spaceId, spaceReplica, dependentOnGroup, lostHosts_); + auto taskRet = genTasks(spaceId, spaceReplica, dependentOnZone, lostHosts_); if (!ok(taskRet)) { LOG(ERROR) << "Generate tasks on space " << std::get<0>(spaceInfo) << " failed"; return error(taskRet); @@ -177,12 +177,12 @@ nebula::cpp2::ErrorCode DataBalanceJobExecutor::buildBalancePlan() { ErrorOr> DataBalanceJobExecutor::genTasks( GraphSpaceID spaceId, int32_t spaceReplica, - bool dependentOnGroup, + bool dependentOnZone, std::vector& lostHosts) { HostParts hostParts; int32_t totalParts = 0; // hostParts is current part allocation map - auto result = getHostParts(spaceId, dependentOnGroup, hostParts, totalParts); + auto result = getHostParts(spaceId, dependentOnZone, hostParts, totalParts); if (!nebula::ok(result)) { return nebula::error(result); } @@ -193,7 +193,7 @@ ErrorOr> DataBalanceJobExecuto return nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND; } - auto fetchHostPartsRet = fetchHostParts(spaceId, dependentOnGroup, hostParts, lostHosts); + auto fetchHostPartsRet = fetchHostParts(spaceId, dependentOnZone, hostParts, lostHosts); if (!nebula::ok(fetchHostPartsRet)) { LOG(ERROR) << "Fetch hosts and parts failed"; return nebula::error(fetchHostPartsRet); @@ -223,7 +223,7 @@ ErrorOr> DataBalanceJobExecuto } auto retCode = - transferLostHost(tasks, confirmedHostParts, lostHost, spaceId, partId, dependentOnGroup); + transferLostHost(tasks, confirmedHostParts, lostHost, spaceId, partId, dependentOnZone); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Transfer lost host " << lostHost << " failed"; return retCode; @@ -232,7 +232,7 @@ ErrorOr> DataBalanceJobExecuto } // 2. Make all hosts in confirmedHostParts balanced - if (balanceParts(spaceId, confirmedHostParts, totalParts, tasks, dependentOnGroup)) { + if (balanceParts(spaceId, confirmedHostParts, totalParts, tasks, dependentOnZone)) { return tasks; } else { return nebula::cpp2::ErrorCode::E_BAD_BALANCE_PLAN; @@ -244,10 +244,10 @@ nebula::cpp2::ErrorCode DataBalanceJobExecutor::transferLostHost(std::vector result; - if (dependentOnGroup) { + if (dependentOnZone) { result = hostWithMinimalPartsForZone(source, confirmedHostParts, partId); } else { result = hostWithMinimalParts(confirmedHostParts, partId); @@ -271,12 +271,12 @@ nebula::cpp2::ErrorCode DataBalanceJobExecutor::transferLostHost(std::vector>> DataBalanceJobExecutor::fetchHostParts(GraphSpaceID spaceId, - bool dependentOnGroup, + bool dependentOnZone, const HostParts& hostParts, std::vector& lostHosts) { ErrorOr> activeHostsRet; - if (dependentOnGroup) { - activeHostsRet = ActiveHostsMan::getActiveHostsWithGroup(kvstore_, spaceId); + if (dependentOnZone) { + activeHostsRet = ActiveHostsMan::getActiveHostsWithZones(kvstore_, spaceId); } else { activeHostsRet = ActiveHostsMan::getActiveHosts(kvstore_); } @@ -306,7 +306,7 @@ bool DataBalanceJobExecutor::balanceParts(GraphSpaceID spaceId, HostParts& confirmedHostParts, int32_t totalParts, std::vector& tasks, - bool dependentOnGroup) { + bool dependentOnZone) { auto avgLoad = static_cast(totalParts) / confirmedHostParts.size(); VLOG(3) << "The expect avg load is " << avgLoad; int32_t minLoad = std::floor(avgLoad); @@ -395,7 +395,7 @@ bool DataBalanceJobExecutor::balanceParts(GraphSpaceID spaceId, return false; } - if (dependentOnGroup) { + if (dependentOnZone) { if (!checkZoneLegal(sourceHost, targetHost)) { LOG(INFO) << "sourceHost " << sourceHost << " targetHost " << targetHost << " not same zone"; @@ -492,7 +492,7 @@ nebula::cpp2::ErrorCode DataBalanceJobExecutor::stop() { } ErrorOr BalanceJobExecutor::getHostParts(GraphSpaceID spaceId, - bool dependentOnGroup, + bool dependentOnZone, HostParts& hostParts, int32_t& totalParts) { folly::SharedMutex::ReadHolder rHolder(LockUtils::spaceLock()); @@ -529,29 +529,20 @@ ErrorOr BalanceJobExecutor::getHostParts(GraphSpa auto properties = MetaKeyUtils::parseSpace(value); if (totalParts != properties.get_partition_num()) { - LOG(ERROR) << "Partition number not equals"; - LOG(ERROR) << totalParts << " : " << properties.get_partition_num(); + LOG(ERROR) << "Partition number not equals " << totalParts << " : " + << properties.get_partition_num(); return false; } int32_t replica = properties.get_replica_factor(); LOG(INFO) << "Replica " << replica; - if (dependentOnGroup && properties.group_name_ref().has_value()) { - auto groupName = *properties.group_name_ref(); - auto groupKey = MetaKeyUtils::groupKey(groupName); - std::string groupValue; - retCode = kvstore_->get(kDefaultSpaceId, kDefaultPartId, groupKey, &groupValue); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Get group " << groupName - << " failed: " << apache::thrift::util::enumNameSafe(retCode); - return retCode; - } - - int32_t zoneSize = MetaKeyUtils::parseZoneNames(std::move(groupValue)).size(); + if (dependentOnZone && !properties.get_zone_names().empty()) { + auto zoneNames = properties.get_zone_names(); + int32_t zoneSize = zoneNames.size(); LOG(INFO) << "Zone Size " << zoneSize; innerBalance_ = (replica == zoneSize); - auto activeHostsRet = ActiveHostsMan::getActiveHostsWithGroup(kvstore_, spaceId); + auto activeHostsRet = ActiveHostsMan::getActiveHostsWithZones(kvstore_, spaceId); if (!nebula::ok(activeHostsRet)) { return nebula::error(activeHostsRet); } @@ -572,9 +563,9 @@ ErrorOr BalanceJobExecutor::getHostParts(GraphSpa confirmedHostParts.erase(h); } - auto zonePartsRet = assembleZoneParts(groupName, confirmedHostParts); + auto zonePartsRet = assembleZoneParts(zoneNames, confirmedHostParts); if (zonePartsRet != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Assemble Zone Parts failed group: " << groupName; + LOG(ERROR) << "Assemble Zone Parts failed"; return zonePartsRet; } } @@ -583,26 +574,15 @@ ErrorOr BalanceJobExecutor::getHostParts(GraphSpa return true; } -nebula::cpp2::ErrorCode BalanceJobExecutor::assembleZoneParts(const std::string& groupName, - HostParts& hostParts) { - LOG(INFO) << "Balancer assembleZoneParts"; - auto groupKey = MetaKeyUtils::groupKey(groupName); - std::string groupValue; - auto retCode = kvstore_->get(kDefaultSpaceId, kDefaultPartId, groupKey, &groupValue); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Get group " << groupName - << " failed: " << apache::thrift::util::enumNameSafe(retCode); - return retCode; - } - +nebula::cpp2::ErrorCode BalanceJobExecutor::assembleZoneParts( + const std::vector& zoneNames, HostParts& hostParts) { // zoneHosts use to record this host belong to zone's hosts std::unordered_map, std::vector> zoneHosts; - auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(groupValue)); for (const auto& zoneName : zoneNames) { LOG(INFO) << "Zone Name: " << zoneName; auto zoneKey = MetaKeyUtils::zoneKey(zoneName); std::string zoneValue; - retCode = kvstore_->get(kDefaultSpaceId, kDefaultPartId, zoneKey, &zoneValue); + auto retCode = kvstore_->get(kDefaultSpaceId, kDefaultPartId, zoneKey, &zoneValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Get zone " << zoneName << " failed: " << apache::thrift::util::enumNameSafe(retCode); @@ -878,10 +858,10 @@ folly::Future LeaderBalanceJobExecutor::executeInternal(HostAddr&& addre for (const auto& spaceInfo : spaces) { auto spaceId = std::get<0>(spaceInfo); auto replicaFactor = std::get<1>(spaceInfo); - auto dependentOnGroup = std::get<2>(spaceInfo); + auto dependentOnZone = std::get<2>(spaceInfo); LeaderBalancePlan plan; auto balanceResult = buildLeaderBalancePlan( - hostLeaderMap_.get(), spaceId, replicaFactor, dependentOnGroup, plan); + hostLeaderMap_.get(), spaceId, replicaFactor, dependentOnZone, plan); if (!nebula::ok(balanceResult) || !nebula::value(balanceResult)) { LOG(ERROR) << "Building leader balance plan failed " << "Space: " << spaceId; @@ -924,7 +904,7 @@ ErrorOr LeaderBalanceJobExecutor::buildLeaderBala HostLeaderMap* hostLeaderMap, GraphSpaceID spaceId, int32_t replicaFactor, - bool dependentOnGroup, + bool dependentOnZone, LeaderBalancePlan& plan, bool useDeviation) { PartAllocation peersMap; @@ -952,7 +932,7 @@ ErrorOr LeaderBalanceJobExecutor::buildLeaderBala int32_t totalParts = 0; HostParts allHostParts; - auto result = getHostParts(spaceId, dependentOnGroup, allHostParts, totalParts); + auto result = getHostParts(spaceId, dependentOnZone, allHostParts, totalParts); if (!nebula::ok(result)) { return nebula::error(result); } else { @@ -977,7 +957,7 @@ ErrorOr LeaderBalanceJobExecutor::buildLeaderBala return false; } - if (dependentOnGroup) { + if (dependentOnZone) { for (auto it = allHostParts.begin(); it != allHostParts.end(); it++) { auto min = it->second.size() / replicaFactor; VLOG(3) << "Host: " << it->first << " Bounds: " << min << " : " << min + 1; diff --git a/src/meta/processors/job/BalanceJobExecutor.h b/src/meta/processors/job/BalanceJobExecutor.h index 0de93f5e16e..3832122fce1 100644 --- a/src/meta/processors/job/BalanceJobExecutor.h +++ b/src/meta/processors/job/BalanceJobExecutor.h @@ -57,7 +57,8 @@ class BalanceJobExecutor : public MetaJobExecutor { std::vector& expand, std::vector& lost); - nebula::cpp2::ErrorCode assembleZoneParts(const std::string& groupName, HostParts& hostParts); + nebula::cpp2::ErrorCode assembleZoneParts(const std::vector& zoneNames, + HostParts& hostParts); nebula::cpp2::ErrorCode save(const std::string& k, const std::string& v); @@ -119,7 +120,7 @@ class DataBalanceJobExecutor : public BalanceJobExecutor { ErrorOr> genTasks( GraphSpaceID spaceId, int32_t spaceReplica, - bool dependentOnGroup, + bool dependentOnZone, std::vector& lostHosts); ErrorOr hostWithMinimalParts(const HostParts& hostParts, PartitionID partId); @@ -131,22 +132,26 @@ class DataBalanceJobExecutor : public BalanceJobExecutor { HostParts& confirmedHostParts, int32_t totalParts, std::vector& tasks, - bool dependentOnGroup); + bool dependentOnZone); + ErrorOr>> fetchHostParts( GraphSpaceID spaceId, bool dependentOnGroup, const HostParts& hostParts, std::vector& lostHosts); + nebula::cpp2::ErrorCode transferLostHost(std::vector& tasks, HostParts& confirmedHostParts, const HostAddr& source, GraphSpaceID spaceId, PartitionID partId, - bool dependentOnGroup); + bool dependentOnZone); + Status checkReplica(const HostParts& hostParts, const std::vector& activeHosts, int32_t replica, PartitionID partId); + std::vector> sortedHostsByParts(const HostParts& hostParts); bool checkZoneLegal(const HostAddr& source, const HostAddr& target); nebula::cpp2::ErrorCode finishInternal(bool ret = true); @@ -200,7 +205,7 @@ class LeaderBalanceJobExecutor : public BalanceJobExecutor { ErrorOr buildLeaderBalancePlan(HostLeaderMap* hostLeaderMap, GraphSpaceID spaceId, int32_t replicaFactor, - bool dependentOnGroup, + bool dependentOnZone, LeaderBalancePlan& plan, bool useDeviation = true); diff --git a/src/meta/processors/parts/CreateSpaceProcessor.cpp b/src/meta/processors/parts/CreateSpaceProcessor.cpp index 87333321eee..c04da56cf88 100644 --- a/src/meta/processors/parts/CreateSpaceProcessor.cpp +++ b/src/meta/processors/parts/CreateSpaceProcessor.cpp @@ -13,8 +13,6 @@ DEFINE_int32(default_replica_factor, 1, "The default replica factor when a space namespace nebula { namespace meta { -const std::string defaultGroup = "default"; // NOLINT - void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { folly::SharedMutex::WriteHolder wHolder(LockUtils::spaceLock()); auto properties = req.get_properties(); @@ -112,23 +110,28 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { data.emplace_back(MetaKeyUtils::spaceKey(spaceId), MetaKeyUtils::spaceVal(properties)); nebula::cpp2::ErrorCode code = nebula::cpp2::ErrorCode::SUCCEEDED; - if (properties.group_name_ref().has_value()) { - auto& groupName = *properties.group_name_ref(); - LOG(INFO) << "Create Space on group: " << groupName; - auto groupKey = MetaKeyUtils::groupKey(groupName); - auto ret = doGet(groupKey); - if (!nebula::ok(ret)) { - auto retCode = nebula::error(ret); - if (retCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { - retCode = nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND; + if (!properties.get_zone_names().empty()) { + auto zones = properties.get_zone_names(); + for (auto& zone : zones) { + auto zoneKey = MetaKeyUtils::zoneKey(zone); + auto ret = doGet(zoneKey); + if (!nebula::ok(ret)) { + auto retCode = nebula::error(ret); + if (retCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + code = nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND; + } + LOG(ERROR) << " Get Zone Name: " << zone << " failed."; + break; } - LOG(ERROR) << " Get Group Name: " << groupName << " failed."; - handleErrorCode(retCode); + } + + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Create space failed"; + handleErrorCode(code); onFinished(); return; } - auto zones = MetaKeyUtils::parseZoneNames(nebula::value(ret)); int32_t zoneNum = zones.size(); if (replicaFactor > zoneNum) { LOG(ERROR) << "Replication number should less than or equal to zone number."; diff --git a/src/meta/processors/parts/GetSpaceProcessor.cpp b/src/meta/processors/parts/GetSpaceProcessor.cpp index 6dbcdcde060..883ed36c144 100644 --- a/src/meta/processors/parts/GetSpaceProcessor.cpp +++ b/src/meta/processors/parts/GetSpaceProcessor.cpp @@ -37,10 +37,8 @@ void GetSpaceProcessor::process(const cpp2::GetSpaceReq& req) { auto properties = MetaKeyUtils::parseSpace(nebula::value(ret)); VLOG(3) << "Get Space SpaceName: " << spaceName << ", Partition Num " << properties.get_partition_num() << ", Replica Factor " - << properties.get_replica_factor(); - if (properties.group_name_ref().has_value()) { - LOG(INFO) << "Space " << spaceName << " is bind to the group " << *properties.group_name_ref(); - } + << properties.get_replica_factor() << ", bind to the zones " + << folly::join(",", properties.get_zone_names()); cpp2::SpaceItem item; item.set_space_id(spaceId); diff --git a/src/meta/processors/zone/UpdateZoneProcessor.cpp b/src/meta/processors/zone/UpdateZoneProcessor.cpp index 767b4b606c3..cb7af515804 100644 --- a/src/meta/processors/zone/UpdateZoneProcessor.cpp +++ b/src/meta/processors/zone/UpdateZoneProcessor.cpp @@ -119,41 +119,52 @@ void DropHostFromZoneProcessor::process(const cpp2::DropHostFromZoneReq& req) { const auto& spacePrefix = MetaKeyUtils::spacePrefix(); auto spaceIterRet = doPrefix(spacePrefix); auto spaceIter = nebula::value(spaceIterRet).get(); + auto code = nebula::cpp2::ErrorCode::SUCCEEDED; while (spaceIter->valid()) { auto spaceId = MetaKeyUtils::spaceId(spaceIter->key()); auto spaceKey = MetaKeyUtils::spaceKey(spaceId); auto ret = doGet(spaceKey); if (!nebula::ok(ret)) { - auto retCode = nebula::error(ret); + code = nebula::error(ret); LOG(ERROR) << "Get Space " << spaceId - << " error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; + << " error: " << apache::thrift::util::enumNameSafe(code); + break; } auto properties = MetaKeyUtils::parseSpace(nebula::value(ret)); - if (!properties.group_name_ref().has_value()) { - spaceIter->next(); - continue; - } const auto& partPrefix = MetaKeyUtils::partPrefix(spaceId); auto partIterRet = doPrefix(partPrefix); auto partIter = nebula::value(partIterRet).get(); + while (partIter->valid()) { auto partHosts = MetaKeyUtils::parsePartVal(partIter->val()); for (auto& h : partHosts) { if (h == req.get_node()) { LOG(ERROR) << h << " is related with partition"; - handleErrorCode(nebula::cpp2::ErrorCode::E_CONFLICT); - onFinished(); - return; + code = nebula::cpp2::ErrorCode::E_CONFLICT; + break; } } - partIter->next(); + + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + break; + } else { + partIter->next(); + } } - spaceIter->next(); + + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + break; + } else { + spaceIter->next(); + } + } + + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(nebula::cpp2::ErrorCode::E_CONFLICT); + onFinished(); + return; } auto hosts = MetaKeyUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); diff --git a/src/meta/test/BalancerTest.cpp b/src/meta/test/BalancerTest.cpp index 2151d3f1341..308e7db25d8 100644 --- a/src/meta/test/BalancerTest.cpp +++ b/src/meta/test/BalancerTest.cpp @@ -167,7 +167,8 @@ TEST(BalanceTest, SimpleTestWithZone) { properties.set_space_name("default_space"); properties.set_partition_num(4); properties.set_replica_factor(3); - properties.set_group_name("group_0"); + std::vector zones = {"zone_0", "zone_1", "zone_2", "zone_3"}; + properties.set_zone_names(std::move(zones)); cpp2::CreateSpaceReq req; req.set_properties(std::move(properties)); auto* processor = CreateSpaceProcessor::instance(kv); @@ -190,7 +191,8 @@ TEST(BalanceTest, SimpleTestWithZone) { JobDescription jd( testJobId.fetch_add(1, std::memory_order_relaxed), cpp2::AdminCmd::DATA_BALANCE, {}); DataBalanceJobExecutor balancer(jd, kv, &client, {}); - auto code = balancer.assembleZoneParts("group_0", hostParts); + std::vector zones = {"zone_0", "zone_1", "zone_2", "zone_3"}; + auto code = balancer.assembleZoneParts(zones, hostParts); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code); balancer.balanceParts(0, hostParts, totalParts, tasks, true); for (auto it = hostParts.begin(); it != hostParts.end(); it++) { @@ -224,7 +226,8 @@ TEST(BalanceTest, ExpansionZoneTest) { properties.set_space_name("default_space"); properties.set_partition_num(4); properties.set_replica_factor(3); - properties.set_group_name("default_group"); + std::vector zones = {"zone_0", "zone_1", "zone_2"}; + properties.set_zone_names(std::move(zones)); cpp2::CreateSpaceReq req; req.set_properties(std::move(properties)); auto* processor = CreateSpaceProcessor::instance(kv); @@ -257,6 +260,22 @@ TEST(BalanceTest, ExpansionZoneTest) { GroupInfo groupInfo = {{"default_group", {"zone_0", "zone_1", "zone_2", "zone_3"}}}; TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); } + { + cpp2::SpaceDesc properties; + properties.set_space_name("default_space"); + properties.set_partition_num(4); + properties.set_replica_factor(3); + std::vector zones = {"zone_0", "zone_1", "zone_2", "zone_3"}; + properties.set_zone_names(std::move(zones)); + std::vector data; + data.emplace_back(MetaKeyUtils::spaceKey(1), MetaKeyUtils::spaceVal(properties)); + folly::Baton baton; + kv->asyncMultiPut(0, 0, std::move(data), [&](nebula::cpp2::ErrorCode code) { + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code); + baton.post(); + }); + baton.wait(); + } { HostParts hostParts; int32_t totalParts = 0; @@ -296,7 +315,8 @@ TEST(BalanceTest, ExpansionHostIntoZoneTest) { properties.set_space_name("default_space"); properties.set_partition_num(4); properties.set_replica_factor(3); - properties.set_group_name("default_group"); + std::vector zones = {"zone_0", "zone_1", "zone_2"}; + properties.set_zone_names(std::move(zones)); cpp2::CreateSpaceReq req; req.set_properties(std::move(properties)); auto* processor = CreateSpaceProcessor::instance(kv); @@ -374,7 +394,8 @@ TEST(BalanceTest, ShrinkZoneTest) { properties.set_space_name("default_space"); properties.set_partition_num(4); properties.set_replica_factor(3); - properties.set_group_name("default_group"); + std::vector zones = {"zone_0", "zone_1", "zone_2", "zone_3"}; + properties.set_zone_names(std::move(zones)); cpp2::CreateSpaceReq req; req.set_properties(std::move(properties)); auto* processor = CreateSpaceProcessor::instance(kv); @@ -424,7 +445,8 @@ TEST(BalanceTest, ShrinkHostFromZoneTest) { properties.set_space_name("default_space"); properties.set_partition_num(4); properties.set_replica_factor(3); - properties.set_group_name("default_group"); + std::vector zones = {"zone_0", "zone_1", "zone_2"}; + properties.set_zone_names(std::move(zones)); cpp2::CreateSpaceReq req; req.set_properties(std::move(properties)); auto* processor = CreateSpaceProcessor::instance(kv); @@ -520,7 +542,8 @@ TEST(BalanceTest, BalanceWithComplexZoneTest) { properties.set_space_name("space_on_group_0"); properties.set_partition_num(64); properties.set_replica_factor(3); - properties.set_group_name("group_0"); + std::vector zones = {"zone_0", "zone_1", "zone_2", "zone_3", "zone_4"}; + properties.set_zone_names(std::move(zones)); cpp2::CreateSpaceReq req; req.set_properties(std::move(properties)); auto* processor = CreateSpaceProcessor::instance(kv); @@ -537,7 +560,9 @@ TEST(BalanceTest, BalanceWithComplexZoneTest) { properties.set_space_name("space_on_group_1"); properties.set_partition_num(81); properties.set_replica_factor(3); - properties.set_group_name("group_1"); + std::vector zones = { + "zone_0", "zone_1", "zone_2", "zone_3", "zone_4", "zone_5", "zone_6", "zone_7", "zone_8"}; + properties.set_zone_names(std::move(zones)); cpp2::CreateSpaceReq req; req.set_properties(std::move(properties)); auto* processor = CreateSpaceProcessor::instance(kv); @@ -568,7 +593,8 @@ TEST(BalanceTest, BalanceWithComplexZoneTest) { int32_t totalParts = 64 * 3; std::vector tasks; auto hostParts = assignHostParts(kv, 2); - auto code = balancer.assembleZoneParts("group_0", hostParts); + std::vector zones = {"zone_0", "zone_1", "zone_2", "zone_3", "zone_4"}; + auto code = balancer.assembleZoneParts(zones, hostParts); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code); balancer.balanceParts(2, hostParts, totalParts, tasks, true); } @@ -605,7 +631,9 @@ TEST(BalanceTest, BalanceWithComplexZoneTest) { int32_t totalParts = 243; std::vector tasks; dump(hostParts, tasks); - auto code = balancer.assembleZoneParts("group_1", hostParts); + std::vector zones = { + "zone_0", "zone_1", "zone_2", "zone_3", "zone_4", "zone_5", "zone_6", "zone_7", "zone_8"}; + auto code = balancer.assembleZoneParts(zones, hostParts); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code); balancer.balanceParts(3, hostParts, totalParts, tasks, true); @@ -1654,7 +1682,8 @@ TEST(BalanceTest, LeaderBalanceWithZoneTest) { properties.set_space_name("default_space"); properties.set_partition_num(8); properties.set_replica_factor(3); - properties.set_group_name("default_group"); + std::vector zones = {"zone_0", "zone_1", "zone_2"}; + properties.set_zone_names(std::move(zones)); cpp2::CreateSpaceReq req; req.set_properties(std::move(properties)); auto* processor = CreateSpaceProcessor::instance(kv); @@ -1732,7 +1761,8 @@ TEST(BalanceTest, LeaderBalanceWithLargerZoneTest) { properties.set_space_name("default_space"); properties.set_partition_num(8); properties.set_replica_factor(3); - properties.set_group_name("default_group"); + std::vector zones = {"zone_0", "zone_1", "zone_2", "zone_3", "zone_4"}; + properties.set_zone_names(std::move(zones)); cpp2::CreateSpaceReq req; req.set_properties(std::move(properties)); auto* processor = CreateSpaceProcessor::instance(kv); @@ -1832,7 +1862,8 @@ TEST(BalanceTest, LeaderBalanceWithComplexZoneTest) { properties.set_space_name("space_on_group_0"); properties.set_partition_num(64); properties.set_replica_factor(3); - properties.set_group_name("group_0"); + std::vector zones = {"zone_0", "zone_1", "zone_2", "zone_3", "zone_4"}; + properties.set_zone_names(std::move(zones)); cpp2::CreateSpaceReq req; req.set_properties(std::move(properties)); auto* processor = CreateSpaceProcessor::instance(kv); @@ -1848,7 +1879,9 @@ TEST(BalanceTest, LeaderBalanceWithComplexZoneTest) { properties.set_space_name("space_on_group_1"); properties.set_partition_num(81); properties.set_replica_factor(3); - properties.set_group_name("group_1"); + std::vector zones = { + "zone_0", "zone_1", "zone_2", "zone_3", "zone_4", "zone_5", "zone_6", "zone_7", "zone_8"}; + properties.set_zone_names(std::move(zones)); cpp2::CreateSpaceReq req; req.set_properties(std::move(properties)); auto* processor = CreateSpaceProcessor::instance(kv); diff --git a/src/meta/test/RestoreProcessorTest.cpp b/src/meta/test/RestoreProcessorTest.cpp index 37cc156507a..fa67ad2df63 100644 --- a/src/meta/test/RestoreProcessorTest.cpp +++ b/src/meta/test/RestoreProcessorTest.cpp @@ -37,19 +37,16 @@ TEST(RestoreProcessorTest, RestoreTest) { bool ret = false; cpp2::SpaceDesc properties; GraphSpaceID id = 1; - std::string groupName = "test_group"; properties.set_space_name("test_space"); int partNum = 10; properties.set_partition_num(partNum); properties.set_replica_factor(3); - properties.set_group_name(groupName); cpp2::SpaceDesc properties2; GraphSpaceID id2 = 2; properties2.set_space_name("test_space2"); properties2.set_partition_num(partNum); properties2.set_replica_factor(3); - properties2.set_group_name(groupName); auto spaceVal = MetaKeyUtils::spaceVal(properties); std::vector data; @@ -74,12 +71,8 @@ TEST(RestoreProcessorTest, RestoreTest) { data.emplace_back(MetaKeyUtils::partKey(id, partId), MetaKeyUtils::partVal(hosts4)); } - auto groupId = 1; std::string zoneName = "test_zone"; std::vector zoneNames = {zoneName}; - data.emplace_back(MetaKeyUtils::indexGroupKey(groupName), - std::string(reinterpret_cast(&groupId), sizeof(GroupID))); - data.emplace_back(MetaKeyUtils::groupKey(groupName), MetaKeyUtils::groupVal(zoneNames)); data.emplace_back(MetaKeyUtils::userKey("root"), MetaKeyUtils::userVal("root")); @@ -262,19 +255,16 @@ TEST(RestoreProcessorTest, RestoreFullTest) { bool ret = false; cpp2::SpaceDesc properties; GraphSpaceID id = 1; - std::string groupName = "test_group"; properties.set_space_name("test_space"); int partNum = 10; properties.set_partition_num(partNum); properties.set_replica_factor(3); - properties.set_group_name(groupName); cpp2::SpaceDesc properties2; GraphSpaceID id2 = 2; properties2.set_space_name("test_space2"); properties2.set_partition_num(partNum); properties2.set_replica_factor(3); - properties2.set_group_name(groupName); auto spaceVal = MetaKeyUtils::spaceVal(properties); std::vector data; @@ -299,13 +289,8 @@ TEST(RestoreProcessorTest, RestoreFullTest) { data.emplace_back(MetaKeyUtils::partKey(id, partId), MetaKeyUtils::partVal(hosts4)); } - auto groupId = 1; std::string zoneName = "test_zone"; std::vector zoneNames = {zoneName}; - data.emplace_back(MetaKeyUtils::indexGroupKey(groupName), - std::string(reinterpret_cast(&groupId), sizeof(GroupID))); - data.emplace_back(MetaKeyUtils::groupKey(groupName), MetaKeyUtils::groupVal(zoneNames)); - data.emplace_back(MetaKeyUtils::userKey("root"), MetaKeyUtils::userVal("root")); auto zoneId = 1; @@ -410,11 +395,6 @@ TEST(RestoreProcessorTest, RestoreFullTest) { iter->next(); } - prefix = MetaKeyUtils::groupPrefix(); - result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); - // ASSERT_NE(nebula::cpp2::ErrorCode::SUCCEEDED, result); - ASSERT_TRUE(iter->valid()); - prefix = MetaKeyUtils::userPrefix(); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); diff --git a/src/parser/AdminSentences.cpp b/src/parser/AdminSentences.cpp index 4e3cfafc281..e09bb6e5811 100644 --- a/src/parser/AdminSentences.cpp +++ b/src/parser/AdminSentences.cpp @@ -91,9 +91,9 @@ std::string CreateSpaceSentence::toString() const { buf += spaceOpts_->toString(); buf += ")"; } - if (groupName_ != nullptr) { + if (zoneNames_ != nullptr) { buf += " ON "; - buf += *groupName_; + buf += zoneNames_->toString(); } if (comment_ != nullptr) { buf += " comment = \""; diff --git a/src/parser/AdminSentences.h b/src/parser/AdminSentences.h index c08c738d39f..b654700066c 100644 --- a/src/parser/AdminSentences.h +++ b/src/parser/AdminSentences.h @@ -301,11 +301,11 @@ class CreateSpaceSentence final : public CreateSentence { const std::string* spaceName() const { return spaceName_.get(); } - const std::string* groupName() const { return groupName_.get(); } + const ZoneNameList* zoneNames() const { return zoneNames_.get(); } void setOpts(SpaceOptList* spaceOpts) { spaceOpts_.reset(spaceOpts); } - void setGroupName(std::string* name) { groupName_.reset(name); } + void setZoneNames(ZoneNameList* names) { zoneNames_.reset(names); } const SpaceOptList* spaceOpts() const { return spaceOpts_.get(); } @@ -324,7 +324,7 @@ class CreateSpaceSentence final : public CreateSentence { private: std::unique_ptr spaceName_; - std::unique_ptr groupName_; + std::unique_ptr zoneNames_; std::unique_ptr spaceOpts_; std::unique_ptr comment_; }; diff --git a/src/parser/MaintainSentences.h b/src/parser/MaintainSentences.h index 64a8363300c..bc5de67c9a2 100644 --- a/src/parser/MaintainSentences.h +++ b/src/parser/MaintainSentences.h @@ -754,36 +754,6 @@ class ShowCreateEdgeIndexSentence : public Sentence { std::unique_ptr indexName_; }; -class ZoneNameList final { - public: - ZoneNameList() = default; - - void addZone(std::string *zone) { zones_.emplace_back(zone); } - - std::vector zoneNames() const { - std::vector result; - result.resize(zones_.size()); - auto get = [](auto &ptr) { return *ptr.get(); }; - std::transform(zones_.begin(), zones_.end(), result.begin(), get); - return result; - } - - std::string toString() const { - std::string buf; - for (const auto &zone : zones_) { - buf += *zone; - buf += ","; - } - if (!zones_.empty()) { - buf.pop_back(); - } - return buf; - } - - private: - std::vector> zones_; -}; - class AddZoneSentence : public Sentence { public: explicit AddZoneSentence(std::string *zoneName, HostList *hosts) { diff --git a/src/parser/Sentence.h b/src/parser/Sentence.h index e943f8ed24a..2464d581715 100644 --- a/src/parser/Sentence.h +++ b/src/parser/Sentence.h @@ -185,6 +185,36 @@ inline std::ostream &operator<<(std::ostream &os, Sentence::Kind kind) { return os << static_cast(kind); } +class ZoneNameList final { + public: + ZoneNameList() = default; + + void addZone(std::string *zone) { zones_.emplace_back(zone); } + + std::vector zoneNames() const { + std::vector result; + result.resize(zones_.size()); + auto get = [](auto &ptr) { return *ptr.get(); }; + std::transform(zones_.begin(), zones_.end(), result.begin(), get); + return result; + } + + std::string toString() const { + std::string buf; + for (const auto &zone : zones_) { + buf += *zone; + buf += ","; + } + if (!zones_.empty()) { + buf.pop_back(); + } + return buf; + } + + private: + std::vector> zones_; +}; + } // namespace nebula #endif // PARSER_SENTENCE_H_ diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 59b567ceda3..9febad281ea 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -120,6 +120,7 @@ static constexpr size_t kCommentLengthLimit = 256; nebula::GroupClause *group_clause; nebula::HostList *host_list; nebula::HostAddr *host_item; + nebula::ZoneNameList *zone_name_list; std::vector *integer_list; nebula::InBoundClause *in_bound_clause; nebula::OutBoundClause *out_bound_clause; @@ -333,6 +334,7 @@ static constexpr size_t kCommentLengthLimit = 256; %type column_spec %type column_spec_list %type column_name_list +%type zone_name_list %type role_type_clause %type acl_item_clause @@ -3279,6 +3281,17 @@ show_config_item } ; +zone_name_list + : name_label { + $$ = new ZoneNameList(); + $$->addZone($1); + } + | zone_name_list COMMA name_label { + $$ = $1; + $$->addZone($3); + } + ; + create_space_sentence : KW_CREATE KW_SPACE opt_if_not_exists name_label { auto sentence = new CreateSpaceSentence($4, $3); @@ -3289,14 +3302,14 @@ create_space_sentence sentence->setComment($5); $$ = sentence; } - | KW_CREATE KW_SPACE opt_if_not_exists name_label KW_ON name_label { + | KW_CREATE KW_SPACE opt_if_not_exists name_label KW_ON zone_name_list { auto sentence = new CreateSpaceSentence($4, $3); - sentence->setGroupName($6); + sentence->setZoneNames($6); $$ = sentence; } - | KW_CREATE KW_SPACE opt_if_not_exists name_label KW_ON name_label comment_prop_assignment { + | KW_CREATE KW_SPACE opt_if_not_exists name_label KW_ON zone_name_list comment_prop_assignment { auto sentence = new CreateSpaceSentence($4, $3); - sentence->setGroupName($6); + sentence->setZoneNames($6); sentence->setComment($7); $$ = sentence; } @@ -3311,15 +3324,15 @@ create_space_sentence sentence->setComment($8); $$ = sentence; } - | KW_CREATE KW_SPACE opt_if_not_exists name_label L_PAREN space_opt_list R_PAREN KW_ON name_label { + | KW_CREATE KW_SPACE opt_if_not_exists name_label L_PAREN space_opt_list R_PAREN KW_ON zone_name_list { auto sentence = new CreateSpaceSentence($4, $3); - sentence->setGroupName($9); + sentence->setZoneNames($9); sentence->setOpts($6); $$ = sentence; } - | KW_CREATE KW_SPACE opt_if_not_exists name_label L_PAREN space_opt_list R_PAREN KW_ON name_label comment_prop_assignment { + | KW_CREATE KW_SPACE opt_if_not_exists name_label L_PAREN space_opt_list R_PAREN KW_ON zone_name_list comment_prop_assignment { auto sentence = new CreateSpaceSentence($4, $3); - sentence->setGroupName($9); + sentence->setZoneNames($9); sentence->setOpts($6); sentence->setComment($10); $$ = sentence; From 46e5842305df023410b448ae4f83a0418d74df5d Mon Sep 17 00:00:00 2001 From: "darion.yaphet" Date: Wed, 20 Oct 2021 02:13:42 +0800 Subject: [PATCH 2/5] Support white list --- .linters/cpp/checkKeyword.py | 4 + src/clients/meta/MetaClient.cpp | 84 +- src/clients/meta/MetaClient.h | 15 +- src/common/graph/Response.h | 2 +- src/common/utils/MetaKeyUtils.cpp | 42 +- src/common/utils/MetaKeyUtils.h | 16 +- src/common/utils/test/MetaKeyUtilsTest.cpp | 27 +- src/graph/executor/CMakeLists.txt | 2 + src/graph/executor/Executor.cpp | 25 +- src/graph/executor/admin/AddHostsExecutor.cpp | 32 + src/graph/executor/admin/AddHostsExecutor.h | 26 + .../executor/admin/DropHostsExecutor.cpp | 31 + src/graph/executor/admin/DropHostsExecutor.h | 26 + src/graph/executor/admin/ShowHostsExecutor.h | 2 +- src/graph/executor/admin/ZoneExecutor.cpp | 47 +- src/graph/executor/admin/ZoneExecutor.h | 32 +- src/graph/planner/plan/Admin.cpp | 12 + src/graph/planner/plan/Admin.h | 169 +-- src/graph/planner/plan/PlanNode.cpp | 20 +- src/graph/planner/plan/PlanNode.h | 9 +- src/graph/service/PermissionCheck.cpp | 9 +- src/graph/validator/AdminValidator.cpp | 40 + src/graph/validator/AdminValidator.h | 24 + src/graph/validator/MaintainValidator.cpp | 51 +- src/graph/validator/MaintainValidator.h | 34 +- src/graph/validator/Validator.cpp | 18 +- src/interface/common.thrift | 2 +- src/interface/meta.thrift | 48 +- src/meta/ActiveHostsMan.cpp | 47 +- src/meta/ActiveHostsMan.h | 2 + src/meta/CMakeLists.txt | 6 +- src/meta/MetaServiceHandler.cpp | 38 +- src/meta/MetaServiceHandler.h | 15 +- src/meta/processors/BaseProcessor-inl.h | 51 +- src/meta/processors/BaseProcessor.h | 11 +- src/meta/processors/Common.h | 1 + src/meta/processors/admin/HBProcessor.cpp | 16 +- .../processors/parts/CreateSpaceProcessor.cpp | 2 +- .../zone/AddHostsIntoZoneProcessor.cpp | 106 ++ .../zone/AddHostsIntoZoneProcessor.h | 29 + .../processors/zone/AddHostsProcessor.cpp | 70 ++ src/meta/processors/zone/AddHostsProcessor.h | 30 + src/meta/processors/zone/AddZoneProcessor.cpp | 119 --- src/meta/processors/zone/AddZoneProcessor.h | 30 - .../processors/zone/DropHostsProcessor.cpp | 157 +++ src/meta/processors/zone/DropHostsProcessor.h | 34 + .../processors/zone/DropZoneProcessor.cpp | 31 - src/meta/processors/zone/DropZoneProcessor.h | 2 - .../processors/zone/RenameZoneProcessor.cpp | 83 ++ .../processors/zone/RenameZoneProcessor.h | 30 + .../processors/zone/UpdateZoneProcessor.cpp | 188 ---- .../processors/zone/UpdateZoneProcessor.h | 42 - src/meta/test/ActiveHostsManTest.cpp | 22 +- src/meta/test/AdminClientTest.cpp | 1 + src/meta/test/BalancerTest.cpp | 69 +- src/meta/test/CMakeLists.txt | 15 - src/meta/test/CreateBackupProcessorTest.cpp | 8 + src/meta/test/GroupZoneTest.cpp | 382 ------- src/meta/test/HBProcessorTest.cpp | 14 +- src/meta/test/MetaClientTest.cpp | 707 ++++++++----- src/meta/test/ProcessorTest.cpp | 961 +++++++++++++++++- src/meta/test/RestoreProcessorTest.cpp | 9 +- src/meta/test/TestUtils.h | 63 +- src/parser/MaintainSentences.cpp | 61 +- src/parser/MaintainSentences.h | 103 +- src/parser/Sentence.h | 9 +- src/parser/parser.yy | 69 +- src/parser/scanner.lex | 5 + src/parser/test/ParserTest.cpp | 68 +- src/parser/test/ScannerTest.cpp | 12 + src/storage/test/KVClientTest.cpp | 9 + src/tools/meta-dump/MetaDumpTool.cpp | 17 - 72 files changed, 3055 insertions(+), 1538 deletions(-) create mode 100644 src/graph/executor/admin/AddHostsExecutor.cpp create mode 100644 src/graph/executor/admin/AddHostsExecutor.h create mode 100644 src/graph/executor/admin/DropHostsExecutor.cpp create mode 100644 src/graph/executor/admin/DropHostsExecutor.h create mode 100644 src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp create mode 100644 src/meta/processors/zone/AddHostsIntoZoneProcessor.h create mode 100644 src/meta/processors/zone/AddHostsProcessor.cpp create mode 100644 src/meta/processors/zone/AddHostsProcessor.h delete mode 100644 src/meta/processors/zone/AddZoneProcessor.cpp delete mode 100644 src/meta/processors/zone/AddZoneProcessor.h create mode 100644 src/meta/processors/zone/DropHostsProcessor.cpp create mode 100644 src/meta/processors/zone/DropHostsProcessor.h create mode 100644 src/meta/processors/zone/RenameZoneProcessor.cpp create mode 100644 src/meta/processors/zone/RenameZoneProcessor.h delete mode 100644 src/meta/processors/zone/UpdateZoneProcessor.cpp delete mode 100644 src/meta/processors/zone/UpdateZoneProcessor.h delete mode 100644 src/meta/test/GroupZoneTest.cpp diff --git a/.linters/cpp/checkKeyword.py b/.linters/cpp/checkKeyword.py index 7d07dda87e4..8484129a684 100755 --- a/.linters/cpp/checkKeyword.py +++ b/.linters/cpp/checkKeyword.py @@ -111,6 +111,10 @@ 'KW_EXPLAIN', 'KW_UNWIND', 'KW_CASE', + 'KW_HOSTS', + 'KW_ZONE', + 'KW_ZONES', + 'KW_RENAME', ] diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index 676b923adbb..98504ab0782 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -780,8 +780,8 @@ Status MetaClient::handleResponse(const RESP& resp) { return Status::Error("Part not existed!"); case nebula::cpp2::ErrorCode::E_USER_NOT_FOUND: return Status::Error("User not existed!"); - case nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND: - return Status::Error("Group not existed!"); + case nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND: + return Status::Error("Machine not existed!"); case nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND: return Status::Error("Zone not existed!"); case nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND: @@ -3036,17 +3036,65 @@ void MetaClient::loadLeader(const std::vector& hostItems, } } -folly::Future> MetaClient::addZone(std::string zoneName, - std::vector nodes) { - cpp2::AddZoneReq req; - req.set_zone_name(std::move(zoneName)); - req.set_nodes(std::move(nodes)); +folly::Future> MetaClient::addHosts(std::vector hosts) { + cpp2::AddHostsReq req; + req.set_hosts(std::move(hosts)); + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_addHosts(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::dropHosts(std::vector hosts) { + cpp2::DropHostsReq req; + req.set_hosts(std::move(hosts)); + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_dropHosts(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::mergeZone(std::vector zones, + std::string zoneName) { + cpp2::MergeZoneReq req; + req.set_zone_name(zoneName); + req.set_zones(zones); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_mergeZone(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} +folly::Future> MetaClient::renameZone(std::string originalZoneName, + std::string zoneName) { + cpp2::RenameZoneReq req; + req.set_original_zone_name(originalZoneName); + req.set_zone_name(zoneName); folly::Promise> promise; auto future = promise.getFuture(); getResponse( std::move(req), - [](auto client, auto request) { return client->future_addZone(request); }, + [](auto client, auto request) { return client->future_renameZone(request); }, [](cpp2::ExecResp&& resp) -> bool { return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; }, @@ -3070,16 +3118,15 @@ folly::Future> MetaClient::dropZone(std::string zoneName) { return future; } -folly::Future> MetaClient::addHostIntoZone(HostAddr node, std::string zoneName) { - cpp2::AddHostIntoZoneReq req; - req.set_node(node); +folly::Future> MetaClient::splitZone( + std::string zoneName, std::unordered_map>) { + cpp2::SplitZoneReq req; req.set_zone_name(zoneName); - folly::Promise> promise; auto future = promise.getFuture(); getResponse( std::move(req), - [](auto client, auto request) { return client->future_addHostIntoZone(request); }, + [](auto client, auto request) { return client->future_splitZone(request); }, [](cpp2::ExecResp&& resp) -> bool { return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; }, @@ -3087,16 +3134,19 @@ folly::Future> MetaClient::addHostIntoZone(HostAddr node, std::st return future; } -folly::Future> MetaClient::dropHostFromZone(HostAddr node, std::string zoneName) { - cpp2::DropHostFromZoneReq req; - req.set_node(node); +folly::Future> MetaClient::addHostsIntoZone(std::vector hosts, + std::string zoneName, + bool isNew) { + cpp2::AddHostsIntoZoneReq req; + req.set_hosts(hosts); req.set_zone_name(zoneName); + req.set_is_new(isNew); folly::Promise> promise; auto future = promise.getFuture(); getResponse( std::move(req), - [](auto client, auto request) { return client->future_dropHostFromZone(request); }, + [](auto client, auto request) { return client->future_addHostsIntoZone(request); }, [](cpp2::ExecResp&& resp) -> bool { return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; }, diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h index fab77aa4381..80f975e0181 100644 --- a/src/clients/meta/MetaClient.h +++ b/src/clients/meta/MetaClient.h @@ -597,13 +597,22 @@ class MetaClient { StatusOr getLeaderInfo(); - folly::Future> addZone(std::string zoneName, std::vector nodes); + folly::Future> addHosts(std::vector hosts); + + folly::Future> dropHosts(std::vector hosts); + + folly::Future> mergeZone(std::vector zones, std::string zoneName); + + folly::Future> renameZone(std::string originalZoneName, std::string zoneName); folly::Future> dropZone(std::string zoneName); - folly::Future> addHostIntoZone(HostAddr node, std::string zoneName); + folly::Future> splitZone( + std::string zoneName, std::unordered_map> zones); - folly::Future> dropHostFromZone(HostAddr node, std::string zoneName); + folly::Future> addHostsIntoZone(std::vector hosts, + std::string zoneName, + bool isNew); folly::Future>> getZone(std::string zoneName); diff --git a/src/common/graph/Response.h b/src/common/graph/Response.h index ba97cb29ce2..f3f0ef08268 100644 --- a/src/common/graph/Response.h +++ b/src/common/graph/Response.h @@ -35,7 +35,7 @@ X(E_TAG_PROP_NOT_FOUND, -10) \ X(E_ROLE_NOT_FOUND, -11) \ X(E_CONFIG_NOT_FOUND, -12) \ - X(E_GROUP_NOT_FOUND, -13) \ + X(E_MACHINE_NOT_FOUND, -13) \ X(E_ZONE_NOT_FOUND, -14) \ X(E_LISTENER_NOT_FOUND, -15) \ X(E_PART_NOT_FOUND, -16) \ diff --git a/src/common/utils/MetaKeyUtils.cpp b/src/common/utils/MetaKeyUtils.cpp index 88f4c5a5b19..4098bde0ccb 100644 --- a/src/common/utils/MetaKeyUtils.cpp +++ b/src/common/utils/MetaKeyUtils.cpp @@ -21,6 +21,7 @@ static const std::unordered_map> syste {"users", {"__users__", true}}, {"hosts", {"__hosts__", false}}, {"versions", {"__versions__", false}}, + {"machines", {"__machines__", false}}, {"snapshots", {"__snapshots__", false}}, {"configs", {"__configs__", true}}, {"groups", {"__groups__", true}}, @@ -58,8 +59,9 @@ static const std::unordered_map< // clang-format off static const std::string kSpacesTable = tableMaps.at("spaces").first; // NOLINT static const std::string kPartsTable = tableMaps.at("parts").first; // NOLINT -static const std::string kHostsTable = systemTableMaps.at("hosts").first; // NOLINT static const std::string kVersionsTable = systemTableMaps.at("versions").first; // NOLINT +static const std::string kHostsTable = systemTableMaps.at("hosts").first; // NOLINT +static const std::string kMachinesTable = systemTableMaps.at("machines").first; // NOLINT static const std::string kTagsTable = tableMaps.at("tags").first; // NOLINT static const std::string kEdgesTable = tableMaps.at("edges").first; // NOLINT static const std::string kIndexesTable = tableMaps.at("indexes").first; // NOLINT @@ -239,6 +241,21 @@ std::vector MetaKeyUtils::parsePartValV2(folly::StringPiece val) { return ret; } +std::string MetaKeyUtils::machineKey(std::string addr, Port port) { + std::string key; + HostAddr h(addr, port); + key.append(kMachinesTable.data(), kMachinesTable.size()) + .append(MetaKeyUtils::serializeHostAddr(h)); + return key; +} + +const std::string& MetaKeyUtils::machinePrefix() { return kMachinesTable; } + +HostAddr MetaKeyUtils::parseMachineKey(folly::StringPiece key) { + key.advance(kMachinesTable.size()); + return MetaKeyUtils::deserializeHostAddr(key); +} + std::string MetaKeyUtils::hostKey(std::string addr, Port port) { return hostKeyV2(addr, port); } std::string MetaKeyUtils::hostKeyV2(std::string addr, Port port) { @@ -961,29 +978,6 @@ MetaKeyUtils::parseBalanceTaskVal(const folly::StringPiece& rawVal) { return std::make_tuple(status, ret, start, end); } -std::string MetaKeyUtils::groupKey(const std::string& group) { - std::string key; - key.reserve(kGroupsTable.size() + group.size()); - key.append(kGroupsTable.data(), kGroupsTable.size()).append(group); - return key; -} - -std::string MetaKeyUtils::groupVal(const std::vector& zones) { - return folly::join(",", zones); -} - -const std::string& MetaKeyUtils::groupPrefix() { return kGroupsTable; } - -std::string MetaKeyUtils::parseGroupName(folly::StringPiece rawData) { - return rawData.subpiece(kGroupsTable.size(), rawData.size()).toString(); -} - -std::vector MetaKeyUtils::parseZoneNames(folly::StringPiece rawData) { - std::vector zones; - folly::split(',', rawData.str(), zones); - return zones; -} - std::string MetaKeyUtils::zoneKey(const std::string& zone) { std::string key; key.reserve(kZonesTable.size() + zone.size()); diff --git a/src/common/utils/MetaKeyUtils.h b/src/common/utils/MetaKeyUtils.h index 7ebce2d009a..3bd831726e6 100644 --- a/src/common/utils/MetaKeyUtils.h +++ b/src/common/utils/MetaKeyUtils.h @@ -110,6 +110,12 @@ class MetaKeyUtils final { static std::vector parsePartValV2(folly::StringPiece val); + static std::string machineKey(std::string ip, Port port); + + static const std::string& machinePrefix(); + + static HostAddr parseMachineKey(folly::StringPiece key); + static std::string hostKey(std::string ip, Port port); static std::string hostKeyV2(std::string addr, Port port); @@ -286,16 +292,6 @@ class MetaKeyUtils final { static std::tuple parseBalanceTaskVal( const folly::StringPiece& rawVal); - static std::string groupKey(const std::string& group); - - static std::string groupVal(const std::vector& zones); - - static const std::string& groupPrefix(); - - static std::string parseGroupName(folly::StringPiece rawData); - - static std::vector parseZoneNames(folly::StringPiece rawData); - static std::string zoneKey(const std::string& zone); static std::string zoneVal(const std::vector& hosts); diff --git a/src/common/utils/test/MetaKeyUtilsTest.cpp b/src/common/utils/test/MetaKeyUtilsTest.cpp index 791c71ca148..fc913671c18 100644 --- a/src/common/utils/test/MetaKeyUtilsTest.cpp +++ b/src/common/utils/test/MetaKeyUtilsTest.cpp @@ -24,8 +24,9 @@ TEST(MetaKeyUtilsTest, SpaceKeyTest) { spaceDesc.set_replica_factor(3); auto spaceVal = MetaKeyUtils::spaceVal(spaceDesc); ASSERT_EQ("default", MetaKeyUtils::spaceName(spaceVal)); - ASSERT_EQ(100, MetaKeyUtils::parseSpace(spaceVal).get_partition_num()); - ASSERT_EQ(3, MetaKeyUtils::parseSpace(spaceVal).get_replica_factor()); + auto properties = MetaKeyUtils::parseSpace(spaceVal); + ASSERT_EQ(100, properties.get_partition_num()); + ASSERT_EQ(3, properties.get_replica_factor()); } TEST(MetaKeyUtilsTest, SpaceKeyWithZonesTest) { @@ -41,8 +42,10 @@ TEST(MetaKeyUtilsTest, SpaceKeyWithZonesTest) { spaceDesc.set_zone_names(std::move(zoneNames)); auto spaceVal = MetaKeyUtils::spaceVal(spaceDesc); ASSERT_EQ("default", MetaKeyUtils::spaceName(spaceVal)); - ASSERT_EQ(100, MetaKeyUtils::parseSpace(spaceVal).get_partition_num()); - ASSERT_EQ(3, MetaKeyUtils::parseSpace(spaceVal).get_replica_factor()); + auto properties = MetaKeyUtils::parseSpace(spaceVal); + ASSERT_EQ(100, properties.get_partition_num()); + ASSERT_EQ(3, properties.get_replica_factor()); + ASSERT_EQ(3, properties.get_zone_names().size()); } TEST(MetaKeyUtilsTest, PartKeyTest) { @@ -92,6 +95,13 @@ TEST(MetaKeyUtilsTest, storeStrIpCodecTest) { } } + { + // kMachinesTable : key + auto key = MetaKeyUtils::machineKey(hostnames[0], ports[0]); + auto host = MetaKeyUtils::parseMachineKey(key); + ASSERT_EQ(host.host, hostnames[0]); + ASSERT_EQ(host.port, ports[0]); + } { // kHostsTable : key auto key = MetaKeyUtils::hostKey(hostnames[0], ports[0]); @@ -200,15 +210,6 @@ TEST(MetaKeyUtilsTest, TagTest) { ASSERT_EQ(parsedSchema, schema); } -TEST(MetaKeyUtilsTest, GroupTest) { - auto groupKey = MetaKeyUtils::groupKey("test_group"); - ASSERT_EQ("test_group", MetaKeyUtils::parseGroupName(groupKey)); - - std::vector zones = {"zone_0", "zone_1", "zone_2"}; - auto groupValue = MetaKeyUtils::groupVal(zones); - ASSERT_EQ(zones, MetaKeyUtils::parseZoneNames(groupValue)); -} - TEST(MetaKeyUtilsTest, ZoneTest) { auto zoneKey = MetaKeyUtils::zoneKey("test_zone"); ASSERT_EQ("test_zone", MetaKeyUtils::parseZoneName(zoneKey)); diff --git a/src/graph/executor/CMakeLists.txt b/src/graph/executor/CMakeLists.txt index 4be67ba0c25..7d6f2ed4923 100644 --- a/src/graph/executor/CMakeLists.txt +++ b/src/graph/executor/CMakeLists.txt @@ -44,6 +44,8 @@ nebula_add_library( algo/ProduceAllPathsExecutor.cpp algo/CartesianProductExecutor.cpp algo/SubgraphExecutor.cpp + admin/AddHostsExecutor.cpp + admin/DropHostsExecutor.cpp admin/SwitchSpaceExecutor.cpp admin/CreateUserExecutor.cpp admin/DropUserExecutor.cpp diff --git a/src/graph/executor/Executor.cpp b/src/graph/executor/Executor.cpp index 78cbaf24573..d52e9bd60ca 100644 --- a/src/graph/executor/Executor.cpp +++ b/src/graph/executor/Executor.cpp @@ -16,12 +16,14 @@ #include "graph/context/ExecutionContext.h" #include "graph/context/QueryContext.h" #include "graph/executor/ExecutionError.h" +#include "graph/executor/admin/AddHostsExecutor.h" #include "graph/executor/admin/ChangePasswordExecutor.h" #include "graph/executor/admin/CharsetExecutor.h" #include "graph/executor/admin/ConfigExecutor.h" #include "graph/executor/admin/CreateUserExecutor.h" #include "graph/executor/admin/DescribeUserExecutor.h" #include "graph/executor/admin/DownloadExecutor.h" +#include "graph/executor/admin/DropHostsExecutor.h" #include "graph/executor/admin/DropUserExecutor.h" #include "graph/executor/admin/GrantRoleExecutor.h" #include "graph/executor/admin/IngestExecutor.h" @@ -442,20 +444,29 @@ Executor *Executor::makeExecutor(QueryContext *qctx, const PlanNode *node) { case PlanNode::Kind::kSubgraph: { return pool->add(new SubgraphExecutor(node, qctx)); } - case PlanNode::Kind::kAddZone: { - return pool->add(new AddZoneExecutor(node, qctx)); + case PlanNode::Kind::kAddHosts: { + return pool->add(new AddHostsExecutor(node, qctx)); + } + case PlanNode::Kind::kDropHosts: { + return pool->add(new DropHostsExecutor(node, qctx)); + } + case PlanNode::Kind::kMergeZone: { + return pool->add(new MergeZoneExecutor(node, qctx)); + } + case PlanNode::Kind::kRenameZone: { + return pool->add(new RenameZoneExecutor(node, qctx)); } case PlanNode::Kind::kDropZone: { return pool->add(new DropZoneExecutor(node, qctx)); } + case PlanNode::Kind::kSplitZone: { + return pool->add(new SplitZoneExecutor(node, qctx)); + } case PlanNode::Kind::kDescribeZone: { return pool->add(new DescribeZoneExecutor(node, qctx)); } - case PlanNode::Kind::kAddHostIntoZone: { - return pool->add(new AddHostIntoZoneExecutor(node, qctx)); - } - case PlanNode::Kind::kDropHostFromZone: { - return pool->add(new DropHostFromZoneExecutor(node, qctx)); + case PlanNode::Kind::kAddHostsIntoZone: { + return pool->add(new AddHostsIntoZoneExecutor(node, qctx)); } case PlanNode::Kind::kShowZones: { return pool->add(new ListZonesExecutor(node, qctx)); diff --git a/src/graph/executor/admin/AddHostsExecutor.cpp b/src/graph/executor/admin/AddHostsExecutor.cpp new file mode 100644 index 00000000000..9ca6da0232e --- /dev/null +++ b/src/graph/executor/admin/AddHostsExecutor.cpp @@ -0,0 +1,32 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "graph/executor/admin/AddHostsExecutor.h" + +#include "graph/planner/plan/Admin.h" + +namespace nebula { +namespace graph { + +folly::Future AddHostsExecutor::execute() { + SCOPED_TIMER(&execTime_); + auto *ahNode = asNode(node()); + return qctx() + ->getMetaClient() + ->addHosts(ahNode->getHosts()) + .via(runner()) + .thenValue([this](StatusOr resp) { + SCOPED_TIMER(&execTime_); + NG_RETURN_IF_ERROR(resp); + if (!resp.value()) { + return Status::Error("Add Hosts failed!"); + } + return Status::OK(); + }); +} + +} // namespace graph +} // namespace nebula diff --git a/src/graph/executor/admin/AddHostsExecutor.h b/src/graph/executor/admin/AddHostsExecutor.h new file mode 100644 index 00000000000..39506591b6a --- /dev/null +++ b/src/graph/executor/admin/AddHostsExecutor.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#ifndef GRAPH_EXECUTOR_ADMIN_ADD_HOST_EXECUTOR_H_ +#define GRAPH_EXECUTOR_ADMIN_ADD_HOST_EXECUTOR_H_ + +#include "graph/executor/Executor.h" + +namespace nebula { +namespace graph { + +class AddHostsExecutor final : public Executor { + public: + AddHostsExecutor(const PlanNode *node, QueryContext *qctx) + : Executor("AddHostsExecutor", node, qctx) {} + + folly::Future execute() override; +}; + +} // namespace graph +} // namespace nebula + +#endif // GRAPH_EXECUTOR_ADMIN_ADD_HOST_EXECUTOR_H_ diff --git a/src/graph/executor/admin/DropHostsExecutor.cpp b/src/graph/executor/admin/DropHostsExecutor.cpp new file mode 100644 index 00000000000..86f775eecf5 --- /dev/null +++ b/src/graph/executor/admin/DropHostsExecutor.cpp @@ -0,0 +1,31 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "graph/executor/admin/DropHostsExecutor.h" + +#include "graph/planner/plan/Admin.h" + +namespace nebula { +namespace graph { + +folly::Future DropHostsExecutor::execute() { + auto *dhNode = asNode(node()); + return qctx() + ->getMetaClient() + ->dropHosts(dhNode->getHosts()) + .via(runner()) + .thenValue([this](StatusOr resp) { + SCOPED_TIMER(&execTime_); + NG_RETURN_IF_ERROR(resp); + if (!resp.value()) { + return Status::Error("Drop Hosts failed!"); + } + return Status::OK(); + }); +} + +} // namespace graph +} // namespace nebula diff --git a/src/graph/executor/admin/DropHostsExecutor.h b/src/graph/executor/admin/DropHostsExecutor.h new file mode 100644 index 00000000000..d8285065063 --- /dev/null +++ b/src/graph/executor/admin/DropHostsExecutor.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#ifndef GRAPH_EXECUTOR_ADMIN_DROP_HOST_EXECUTOR_H_ +#define GRAPH_EXECUTOR_ADMIN_DROP_HOST_EXECUTOR_H_ + +#include "graph/executor/Executor.h" + +namespace nebula { +namespace graph { + +class DropHostsExecutor final : public Executor { + public: + DropHostsExecutor(const PlanNode *node, QueryContext *qctx) + : Executor("DropHostsExecutor", node, qctx) {} + + folly::Future execute() override; +}; + +} // namespace graph +} // namespace nebula + +#endif // GRAPH_EXECUTOR_ADMIN_DROP_HOST_EXECUTOR_H_ diff --git a/src/graph/executor/admin/ShowHostsExecutor.h b/src/graph/executor/admin/ShowHostsExecutor.h index 0761ded6512..3e53f4c118a 100644 --- a/src/graph/executor/admin/ShowHostsExecutor.h +++ b/src/graph/executor/admin/ShowHostsExecutor.h @@ -25,4 +25,4 @@ class ShowHostsExecutor final : public Executor { } // namespace graph } // namespace nebula -#endif +#endif // GRAPH_EXECUTOR_ADMIN_SHOW_HOSTS_EXECUTOR_H_ diff --git a/src/graph/executor/admin/ZoneExecutor.cpp b/src/graph/executor/admin/ZoneExecutor.cpp index a70961b919f..3a5e9938039 100644 --- a/src/graph/executor/admin/ZoneExecutor.cpp +++ b/src/graph/executor/admin/ZoneExecutor.cpp @@ -11,20 +11,14 @@ namespace nebula { namespace graph { -folly::Future AddZoneExecutor::execute() { +folly::Future MergeZoneExecutor::execute() { SCOPED_TIMER(&execTime_); - auto *azNode = asNode(node()); - return qctx() - ->getMetaClient() - ->addZone(azNode->zoneName(), azNode->addresses()) - .via(runner()) - .thenValue([](StatusOr resp) { - if (!resp.ok()) { - LOG(ERROR) << "Add Zone Failed :" << resp.status(); - return resp.status(); - } - return Status::OK(); - }); + return Status::OK(); +} + +folly::Future RenameZoneExecutor::execute() { + SCOPED_TIMER(&execTime_); + return Status::OK(); } folly::Future DropZoneExecutor::execute() { @@ -43,6 +37,11 @@ folly::Future DropZoneExecutor::execute() { }); } +folly::Future SplitZoneExecutor::execute() { + SCOPED_TIMER(&execTime_); + return Status::OK(); +} + folly::Future DescribeZoneExecutor::execute() { SCOPED_TIMER(&execTime_); auto *dzNode = asNode(node()); @@ -70,12 +69,12 @@ folly::Future DescribeZoneExecutor::execute() { }); } -folly::Future AddHostIntoZoneExecutor::execute() { +folly::Future AddHostsIntoZoneExecutor::execute() { SCOPED_TIMER(&execTime_); - auto *ahNode = asNode(node()); + auto *ahNode = asNode(node()); return qctx() ->getMetaClient() - ->addHostIntoZone(ahNode->address(), ahNode->zoneName()) + ->addHostsIntoZone(ahNode->address(), ahNode->zoneName(), ahNode->isNew()) .via(runner()) .thenValue([](StatusOr resp) { if (!resp.ok()) { @@ -86,22 +85,6 @@ folly::Future AddHostIntoZoneExecutor::execute() { }); } -folly::Future DropHostFromZoneExecutor::execute() { - SCOPED_TIMER(&execTime_); - auto *dhNode = asNode(node()); - return qctx() - ->getMetaClient() - ->dropHostFromZone(dhNode->address(), dhNode->zoneName()) - .via(runner()) - .thenValue([](StatusOr resp) { - if (!resp.ok()) { - LOG(ERROR) << "Drop Host From Zone Failed: " << resp.status(); - return resp.status(); - } - return Status::OK(); - }); -} - folly::Future ListZonesExecutor::execute() { SCOPED_TIMER(&execTime_); return qctx()->getMetaClient()->listZones().via(runner()).thenValue( diff --git a/src/graph/executor/admin/ZoneExecutor.h b/src/graph/executor/admin/ZoneExecutor.h index 38e661eb55c..dce98c871d9 100644 --- a/src/graph/executor/admin/ZoneExecutor.h +++ b/src/graph/executor/admin/ZoneExecutor.h @@ -11,10 +11,18 @@ namespace nebula { namespace graph { -class AddZoneExecutor final : public Executor { +class MergeZoneExecutor final : public Executor { public: - AddZoneExecutor(const PlanNode *node, QueryContext *qctx) - : Executor("AddZoneExecutor", node, qctx) {} + MergeZoneExecutor(const PlanNode *node, QueryContext *qctx) + : Executor("MergeZoneExecutor", node, qctx) {} + + folly::Future execute() override; +}; + +class RenameZoneExecutor final : public Executor { + public: + RenameZoneExecutor(const PlanNode *node, QueryContext *qctx) + : Executor("RenameZoneExecutor", node, qctx) {} folly::Future execute() override; }; @@ -27,26 +35,26 @@ class DropZoneExecutor final : public Executor { folly::Future execute() override; }; -class DescribeZoneExecutor final : public Executor { +class SplitZoneExecutor final : public Executor { public: - DescribeZoneExecutor(const PlanNode *node, QueryContext *qctx) - : Executor("DescribeZoneExecutor", node, qctx) {} + SplitZoneExecutor(const PlanNode *node, QueryContext *qctx) + : Executor("SplitZoneExecutor", node, qctx) {} folly::Future execute() override; }; -class AddHostIntoZoneExecutor final : public Executor { +class DescribeZoneExecutor final : public Executor { public: - AddHostIntoZoneExecutor(const PlanNode *node, QueryContext *qctx) - : Executor("AddHostIntoZoneExecutor", node, qctx) {} + DescribeZoneExecutor(const PlanNode *node, QueryContext *qctx) + : Executor("DescribeZoneExecutor", node, qctx) {} folly::Future execute() override; }; -class DropHostFromZoneExecutor final : public Executor { +class AddHostsIntoZoneExecutor final : public Executor { public: - DropHostFromZoneExecutor(const PlanNode *node, QueryContext *qctx) - : Executor("DropHostFromZoneExecutor", node, qctx) {} + AddHostsIntoZoneExecutor(const PlanNode *node, QueryContext *qctx) + : Executor("AddHostsIntoZoneExecutor", node, qctx) {} folly::Future execute() override; }; diff --git a/src/graph/planner/plan/Admin.cpp b/src/graph/planner/plan/Admin.cpp index e48793939fb..d84509fa8de 100644 --- a/src/graph/planner/plan/Admin.cpp +++ b/src/graph/planner/plan/Admin.cpp @@ -92,6 +92,18 @@ std::unique_ptr DropNode::explain() const { return desc; } +std::unique_ptr AddHosts::explain() const { + auto desc = SingleDependencyNode::explain(); + addDescription("hosts", folly::toJson(util::toJson(hosts_)), desc.get()); + return desc; +} + +std::unique_ptr DropHosts::explain() const { + auto desc = SingleDependencyNode::explain(); + addDescription("hosts", folly::toJson(util::toJson(hosts_)), desc.get()); + return desc; +} + std::unique_ptr CreateUser::explain() const { auto desc = CreateNode::explain(); addDescription("username", *username_, desc.get()); diff --git a/src/graph/planner/plan/Admin.h b/src/graph/planner/plan/Admin.h index 1f5b4c24215..c95ecfadf2f 100644 --- a/src/graph/planner/plan/Admin.h +++ b/src/graph/planner/plan/Admin.h @@ -49,6 +49,40 @@ class DropNode : public SingleDependencyNode { bool ifExist_{false}; }; +class AddHosts final : public SingleDependencyNode { + public: + static AddHosts* make(QueryContext* qctx, PlanNode* dep, std::vector hosts) { + return qctx->objPool()->add(new AddHosts(qctx, dep, hosts)); + } + + std::vector getHosts() const { return hosts_; } + + std::unique_ptr explain() const override; + + private: + AddHosts(QueryContext* qctx, PlanNode* dep, std::vector hosts) + : SingleDependencyNode(qctx, Kind::kAddHosts, dep), hosts_(hosts) {} + + std::vector hosts_; +}; + +class DropHosts final : public SingleDependencyNode { + public: + static DropHosts* make(QueryContext* qctx, PlanNode* dep, std::vector hosts) { + return qctx->objPool()->add(new DropHosts(qctx, dep, hosts)); + } + + std::vector getHosts() const { return hosts_; } + + std::unique_ptr explain() const override; + + private: + DropHosts(QueryContext* qctx, PlanNode* dep, std::vector hosts) + : SingleDependencyNode(qctx, Kind::kDropHosts, dep), hosts_(hosts) {} + + std::vector hosts_; +}; + class ShowHosts final : public SingleDependencyNode { // TODO(shylock) meta/storage/graph enumerate public: @@ -772,85 +806,97 @@ class ShowCollation final : public SingleDependencyNode { : SingleDependencyNode(qctx, Kind::kShowCollation, input) {} }; -class AddHostIntoZone final : public SingleDependencyNode { +class AddHostsIntoZone final : public SingleDependencyNode { public: - static AddHostIntoZone* make(QueryContext* qctx, - PlanNode* input, - std::string zoneName, - HostAddr addresses) { + static AddHostsIntoZone* make(QueryContext* qctx, + PlanNode* input, + std::string zoneName, + std::vector addresses, + bool isNew) { return qctx->objPool()->add( - new AddHostIntoZone(qctx, input, std::move(zoneName), std::move(addresses))); + new AddHostsIntoZone(qctx, input, std::move(zoneName), std::move(addresses), isNew)); } const std::string& zoneName() const { return zoneName_; } - const HostAddr& address() const { return addresses_; } + const std::vector& address() const { return addresses_; } + + bool isNew() const { return isNew_; } private: - AddHostIntoZone(QueryContext* qctx, PlanNode* input, std::string zoneName, HostAddr addresses) - : SingleDependencyNode(qctx, Kind::kAddHostIntoZone, input) { + AddHostsIntoZone(QueryContext* qctx, + PlanNode* input, + std::string zoneName, + std::vector addresses, + bool isNew) + : SingleDependencyNode(qctx, Kind::kAddHostsIntoZone, input) { zoneName_ = std::move(zoneName); addresses_ = std::move(addresses); + isNew_ = isNew; } private: std::string zoneName_; - HostAddr addresses_; + std::vector addresses_; + bool isNew_; }; -class DropHostFromZone final : public SingleDependencyNode { +class MergeZone final : public SingleDependencyNode { public: - static DropHostFromZone* make(QueryContext* qctx, - PlanNode* input, - std::string zoneName, - HostAddr addresses) { + static MergeZone* make(QueryContext* qctx, + PlanNode* input, + std::string zoneName, + std::vector zoneNames) { return qctx->objPool()->add( - new DropHostFromZone(qctx, input, std::move(zoneName), std::move(addresses))); + new MergeZone(qctx, input, std::move(zoneName), std::move(zoneNames))); } const std::string& zoneName() const { return zoneName_; } - const HostAddr& address() const { return addresses_; } + const std::vector& zones() const { return zones_; } private: - DropHostFromZone(QueryContext* qctx, PlanNode* input, std::string zoneName, HostAddr addresses) - : SingleDependencyNode(qctx, Kind::kDropHostFromZone, input) { + MergeZone(QueryContext* qctx, + PlanNode* input, + std::string zoneName, + std::vector zoneNames) + : SingleDependencyNode(qctx, Kind::kMergeZone, input) { zoneName_ = std::move(zoneName); - addresses_ = std::move(addresses); + zones_ = std::move(zoneNames); } private: std::string zoneName_; - HostAddr addresses_; + std::vector zones_; }; -class AddZone final : public SingleDependencyNode { +class RenameZone final : public SingleDependencyNode { public: - static AddZone* make(QueryContext* qctx, - PlanNode* input, - std::string zoneName, - std::vector addresses) { + static RenameZone* make(QueryContext* qctx, + PlanNode* input, + std::string originalZoneName, + std::string zoneName) { return qctx->objPool()->add( - new AddZone(qctx, input, std::move(zoneName), std::move(addresses))); + new RenameZone(qctx, input, std::move(originalZoneName), std::move(zoneName))); } - const std::string& zoneName() const { return zoneName_; } + const std::string& originalZoneName() const { return originalZoneName_; } - const std::vector& addresses() const { return addresses_; } + const std::string& zoneName() const { return zoneName_; } private: - AddZone(QueryContext* qctx, - PlanNode* input, - std::string zoneName, - std::vector addresses) - : SingleDependencyNode(qctx, Kind::kAddZone, input) { + RenameZone(QueryContext* qctx, + PlanNode* input, + std::string originalZoneName, + std::string zoneName) + : SingleDependencyNode(qctx, Kind::kRenameZone, input) { + originalZoneName_ = std::move(originalZoneName); zoneName_ = std::move(zoneName); - addresses_ = std::move(addresses); } private: + std::string originalZoneName_; std::string zoneName_; - std::vector addresses_; }; class DropZone final : public SingleDependencyNode { @@ -871,6 +917,35 @@ class DropZone final : public SingleDependencyNode { std::string zoneName_; }; +class SplitZone final : public SingleDependencyNode { + public: + static SplitZone* make(QueryContext* qctx, + PlanNode* input, + std::string zoneName, + std::vector zoneNames) { + return qctx->objPool()->add( + new SplitZone(qctx, input, std::move(zoneName), std::move(zoneNames))); + } + + const std::string& zoneName() const { return zoneName_; } + + const std::vector& zones() const { return zones_; } + + private: + SplitZone(QueryContext* qctx, + PlanNode* input, + std::string zoneName, + std::vector zoneNames) + : SingleDependencyNode(qctx, Kind::kSplitZone, input) { + zoneName_ = std::move(zoneName); + zones_ = std::move(zoneNames); + } + + private: + std::string zoneName_; + std::vector zones_; +}; + class DescribeZone final : public SingleDependencyNode { public: static DescribeZone* make(QueryContext* qctx, PlanNode* input, std::string zoneName) { @@ -889,28 +964,6 @@ class DescribeZone final : public SingleDependencyNode { std::string zoneName_; }; -// class DrainZone final : public SingleDependencyNode { -// public: -// static DrainZone* make(QueryContext* qctx, PlanNode* input, std::string -// zoneName) { -// return qctx->objPool()->add(new DrainZone(qctx, input, -// std::move(zoneName))); -// } - -// const std::string& zoneName() const { -// return zoneName_; -// } - -// private: -// DrainZone(QueryContext* qctx, PlanNode* input, std::string zoneName) -// : SingleDependencyNode(qctx, Kind::kDrainZone, input) { -// zoneName_ = std::move(zoneName); -// } - -// private: -// std::string zoneName_; -// }; - class ListZones final : public SingleDependencyNode { public: static ListZones* make(QueryContext* qctx, PlanNode* input) { diff --git a/src/graph/planner/plan/PlanNode.cpp b/src/graph/planner/plan/PlanNode.cpp index a1629320279..6ac02659611 100644 --- a/src/graph/planner/plan/PlanNode.cpp +++ b/src/graph/planner/plan/PlanNode.cpp @@ -236,17 +236,23 @@ const char* PlanNode::toString(PlanNode::Kind kind) { return "CartesianProduct"; case Kind::kSubgraph: return "Subgraph"; - // Group and Zone - case Kind::kAddZone: - return "AddZone"; + case Kind::kAddHosts: + return "AddHosts"; + case Kind::kDropHosts: + return "DropHosts"; + // Zone + case Kind::kMergeZone: + return "MergeZone"; + case Kind::kRenameZone: + return "RenameZone"; case Kind::kDropZone: return "DropZone"; + case Kind::kSplitZone: + return "SplitZone"; case Kind::kDescribeZone: return "DescribeZone"; - case Kind::kAddHostIntoZone: - return "AddHostIntoZone"; - case Kind::kDropHostFromZone: - return "DropHostFromZone"; + case Kind::kAddHostsIntoZone: + return "AddHostsIntoZone"; case Kind::kShowZones: return "ShowZones"; case Kind::kAddListener: diff --git a/src/graph/planner/plan/PlanNode.h b/src/graph/planner/plan/PlanNode.h index bac87810a32..08ab4ae0466 100644 --- a/src/graph/planner/plan/PlanNode.h +++ b/src/graph/planner/plan/PlanNode.h @@ -149,11 +149,14 @@ class PlanNode { // zone related kShowZones, - kAddZone, + kMergeZone, + kRenameZone, kDropZone, + kSplitZone, + kAddHosts, + kDropHosts, kDescribeZone, - kAddHostIntoZone, - kDropHostFromZone, + kAddHostsIntoZone, // listener related kAddListener, diff --git a/src/graph/service/PermissionCheck.cpp b/src/graph/service/PermissionCheck.cpp index ccc3ebdcf36..13b5859fbc7 100644 --- a/src/graph/service/PermissionCheck.cpp +++ b/src/graph/service/PermissionCheck.cpp @@ -55,12 +55,15 @@ Status PermissionCheck::permissionCheck(ClientSession *session, case Sentence::Kind::kDropSpace: case Sentence::Kind::kCreateSnapshot: case Sentence::Kind::kDropSnapshot: - case Sentence::Kind::kAddZone: + case Sentence::Kind::kAddHosts: + case Sentence::Kind::kDropHosts: + case Sentence::Kind::kMergeZone: + case Sentence::Kind::kRenameZone: case Sentence::Kind::kDropZone: + case Sentence::Kind::kSplitZone: case Sentence::Kind::kDescribeZone: case Sentence::Kind::kListZones: - case Sentence::Kind::kAddHostIntoZone: - case Sentence::Kind::kDropHostFromZone: + case Sentence::Kind::kAddHostsIntoZone: case Sentence::Kind::kShowConfigs: case Sentence::Kind::kSetConfig: case Sentence::Kind::kGetConfig: diff --git a/src/graph/validator/AdminValidator.cpp b/src/graph/validator/AdminValidator.cpp index e71fcdc3892..fb32c5d21ad 100644 --- a/src/graph/validator/AdminValidator.cpp +++ b/src/graph/validator/AdminValidator.cpp @@ -291,6 +291,46 @@ Status ShowListenerValidator::toPlan() { return Status::OK(); } +Status AddHostsValidator::validateImpl() { return Status::OK(); } + +Status AddHostsValidator::toPlan() { + auto sentence = static_cast(sentence_); + auto hosts = sentence->hosts()->hosts(); + if (hosts.empty()) { + return Status::SemanticError("Host is empty"); + } + + auto it = std::unique(hosts.begin(), hosts.end()); + if (it != hosts.end()) { + return Status::SemanticError("Host have duplicated"); + } + + auto *addHost = AddHosts::make(qctx_, nullptr, hosts); + root_ = addHost; + tail_ = root_; + return Status::OK(); +} + +Status DropHostsValidator::validateImpl() { return Status::OK(); } + +Status DropHostsValidator::toPlan() { + auto sentence = static_cast(sentence_); + auto hosts = sentence->hosts()->hosts(); + if (hosts.empty()) { + return Status::SemanticError("Host is empty"); + } + + auto it = std::unique(hosts.begin(), hosts.end()); + if (it != hosts.end()) { + return Status::SemanticError("Host have duplicated"); + } + + auto *dropHost = DropHosts::make(qctx_, nullptr, hosts); + root_ = dropHost; + tail_ = root_; + return Status::OK(); +} + Status ShowHostsValidator::validateImpl() { return Status::OK(); } Status ShowHostsValidator::toPlan() { diff --git a/src/graph/validator/AdminValidator.h b/src/graph/validator/AdminValidator.h index 48c96856edc..0ae1864b9f3 100644 --- a/src/graph/validator/AdminValidator.h +++ b/src/graph/validator/AdminValidator.h @@ -167,6 +167,30 @@ class ShowListenerValidator final : public Validator { Status toPlan() override; }; +class AddHostsValidator final : public Validator { + public: + AddHostsValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { + setNoSpaceRequired(); + } + + private: + Status validateImpl() override; + + Status toPlan() override; +}; + +class DropHostsValidator final : public Validator { + public: + DropHostsValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { + setNoSpaceRequired(); + } + + private: + Status validateImpl() override; + + Status toPlan() override; +}; + class ShowHostsValidator final : public Validator { public: ShowHostsValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { diff --git a/src/graph/validator/MaintainValidator.cpp b/src/graph/validator/MaintainValidator.cpp index 609479c1348..61f30013106 100644 --- a/src/graph/validator/MaintainValidator.cpp +++ b/src/graph/validator/MaintainValidator.cpp @@ -460,11 +460,23 @@ Status ShowEdgeIndexStatusValidator::toPlan() { return Status::OK(); } -Status AddZoneValidator::validateImpl() { return Status::OK(); } +Status MergeZoneValidator::validateImpl() { return Status::OK(); } -Status AddZoneValidator::toPlan() { - auto sentence = static_cast(sentence_); - auto *doNode = AddZone::make(qctx_, nullptr, *sentence->zoneName(), sentence->hosts()->hosts()); +Status MergeZoneValidator::toPlan() { + auto sentence = static_cast(sentence_); + auto *doNode = + MergeZone::make(qctx_, nullptr, *sentence->zoneName(), sentence->zoneNames()->zoneNames()); + root_ = doNode; + tail_ = root_; + return Status::OK(); +} + +Status RenameZoneValidator::validateImpl() { return Status::OK(); } + +Status RenameZoneValidator::toPlan() { + auto sentence = static_cast(sentence_); + auto *doNode = + RenameZone::make(qctx_, nullptr, *sentence->originalZoneName(), *sentence->zoneName()); root_ = doNode; tail_ = root_; return Status::OK(); @@ -480,6 +492,17 @@ Status DropZoneValidator::toPlan() { return Status::OK(); } +Status SplitZoneValidator::validateImpl() { return Status::OK(); } + +Status SplitZoneValidator::toPlan() { + auto sentence = static_cast(sentence_); + auto *doNode = + SplitZone::make(qctx_, nullptr, *sentence->zoneName(), sentence->zoneNames()->zoneNames()); + root_ = doNode; + tail_ = root_; + return Status::OK(); +} + Status DescribeZoneValidator::validateImpl() { return Status::OK(); } Status DescribeZoneValidator::toPlan() { @@ -499,22 +522,12 @@ Status ListZonesValidator::toPlan() { return Status::OK(); } -Status AddHostIntoZoneValidator::validateImpl() { return Status::OK(); } +Status AddHostsIntoZoneValidator::validateImpl() { return Status::OK(); } -Status AddHostIntoZoneValidator::toPlan() { - auto sentence = static_cast(sentence_); - auto *doNode = AddHostIntoZone::make(qctx_, nullptr, *sentence->zoneName(), *sentence->address()); - root_ = doNode; - tail_ = root_; - return Status::OK(); -} - -Status DropHostFromZoneValidator::validateImpl() { return Status::OK(); } - -Status DropHostFromZoneValidator::toPlan() { - auto sentence = static_cast(sentence_); - auto *doNode = - DropHostFromZone::make(qctx_, nullptr, *sentence->zoneName(), *sentence->address()); +Status AddHostsIntoZoneValidator::toPlan() { + auto sentence = static_cast(sentence_); + auto *doNode = AddHostsIntoZone::make( + qctx_, nullptr, *sentence->zoneName(), sentence->address()->hosts(), sentence->isNew()); root_ = doNode; tail_ = root_; return Status::OK(); diff --git a/src/graph/validator/MaintainValidator.h b/src/graph/validator/MaintainValidator.h index 865994424ad..4c577063536 100644 --- a/src/graph/validator/MaintainValidator.h +++ b/src/graph/validator/MaintainValidator.h @@ -13,6 +13,7 @@ namespace nebula { namespace graph { + class CreateTagValidator final : public Validator { public: CreateTagValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) {} @@ -310,9 +311,21 @@ class ShowEdgeIndexStatusValidator final : public Validator { Status toPlan() override; }; -class AddZoneValidator final : public Validator { +class MergeZoneValidator final : public Validator { + public: + MergeZoneValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { + setNoSpaceRequired(); + } + + private: + Status validateImpl() override; + + Status toPlan() override; +}; + +class RenameZoneValidator final : public Validator { public: - AddZoneValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { + RenameZoneValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { setNoSpaceRequired(); } @@ -334,9 +347,9 @@ class DropZoneValidator final : public Validator { Status toPlan() override; }; -class DescribeZoneValidator final : public Validator { +class SplitZoneValidator final : public Validator { public: - DescribeZoneValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { + SplitZoneValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { setNoSpaceRequired(); } @@ -346,9 +359,9 @@ class DescribeZoneValidator final : public Validator { Status toPlan() override; }; -class ListZonesValidator final : public Validator { +class DescribeZoneValidator final : public Validator { public: - ListZonesValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { + DescribeZoneValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { setNoSpaceRequired(); } @@ -358,10 +371,9 @@ class ListZonesValidator final : public Validator { Status toPlan() override; }; -class AddHostIntoZoneValidator final : public Validator { +class ListZonesValidator final : public Validator { public: - AddHostIntoZoneValidator(Sentence* sentence, QueryContext* context) - : Validator(sentence, context) { + ListZonesValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { setNoSpaceRequired(); } @@ -371,9 +383,9 @@ class AddHostIntoZoneValidator final : public Validator { Status toPlan() override; }; -class DropHostFromZoneValidator final : public Validator { +class AddHostsIntoZoneValidator final : public Validator { public: - DropHostFromZoneValidator(Sentence* sentence, QueryContext* context) + AddHostsIntoZoneValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) { setNoSpaceRequired(); } diff --git a/src/graph/validator/Validator.cpp b/src/graph/validator/Validator.cpp index 687d4731492..022e60a023c 100644 --- a/src/graph/validator/Validator.cpp +++ b/src/graph/validator/Validator.cpp @@ -156,6 +156,10 @@ std::unique_ptr Validator::makeValidator(Sentence* sentence, QueryCon return std::make_unique(sentence, context); case Sentence::Kind::kUpdateEdge: return std::make_unique(sentence, context); + case Sentence::Kind::kAddHosts: + return std::make_unique(sentence, context); + case Sentence::Kind::kDropHosts: + return std::make_unique(sentence, context); case Sentence::Kind::kShowHosts: return std::make_unique(sentence, context); case Sentence::Kind::kShowMetaLeader: @@ -202,18 +206,20 @@ std::unique_ptr Validator::makeValidator(Sentence* sentence, QueryCon return std::make_unique(sentence, context); case Sentence::Kind::kLookup: return std::make_unique(sentence, context); - case Sentence::Kind::kAddZone: - return std::make_unique(sentence, context); + case Sentence::Kind::kMergeZone: + return std::make_unique(sentence, context); + case Sentence::Kind::kRenameZone: + return std::make_unique(sentence, context); case Sentence::Kind::kDropZone: return std::make_unique(sentence, context); + case Sentence::Kind::kSplitZone: + return std::make_unique(sentence, context); case Sentence::Kind::kDescribeZone: return std::make_unique(sentence, context); case Sentence::Kind::kListZones: return std::make_unique(sentence, context); - case Sentence::Kind::kAddHostIntoZone: - return std::make_unique(sentence, context); - case Sentence::Kind::kDropHostFromZone: - return std::make_unique(sentence, context); + case Sentence::Kind::kAddHostsIntoZone: + return std::make_unique(sentence, context); case Sentence::Kind::kAddListener: return std::make_unique(sentence, context); case Sentence::Kind::kRemoveListener: diff --git a/src/interface/common.thrift b/src/interface/common.thrift index abcc25faeca..8fa6ed9a49e 100644 --- a/src/interface/common.thrift +++ b/src/interface/common.thrift @@ -311,7 +311,7 @@ enum ErrorCode { E_TAG_PROP_NOT_FOUND = -10, E_ROLE_NOT_FOUND = -11, E_CONFIG_NOT_FOUND = -12, - E_GROUP_NOT_FOUND = -13, + E_MACHINE_NOT_FOUND = -13, E_ZONE_NOT_FOUND = -14, E_LISTENER_NOT_FOUND = -15, E_PART_NOT_FOUND = -16, diff --git a/src/interface/meta.thrift b/src/interface/meta.thrift index 1cb2bd43446..6b3fe96a08c 100644 --- a/src/interface/meta.thrift +++ b/src/interface/meta.thrift @@ -432,6 +432,14 @@ struct ListEdgesResp { 3: list edges, } +struct AddHostsReq { + 1: list hosts, +} + +struct DropHostsReq { + 1: list hosts, +} + enum ListHostType { // nebula 1.0 show hosts, show leader, partition info ALLOC = 0x00, @@ -815,23 +823,28 @@ struct ListIndexStatusResp { } // Zone related interface -struct AddZoneReq { - 1: binary zone_name, - 2: list nodes, +struct MergeZoneReq { + 1: list zones, + 2: binary zone_name, } struct DropZoneReq { 1: binary zone_name, } -struct AddHostIntoZoneReq { - 1: common.HostAddr node, - 2: binary zone_name, +struct SplitZoneReq { + 1: binary zone_name, +} + +struct RenameZoneReq { + 1: binary original_zone_name, + 2: binary zone_name, } -struct DropHostFromZoneReq { - 1: common.HostAddr node, - 2: binary zone_name, +struct AddHostsIntoZoneReq { + 1: list hosts, + 2: binary zone_name, + 3: bool is_new, } struct GetZoneReq { @@ -1134,7 +1147,10 @@ service MetaService { GetEdgeResp getEdge(1: GetEdgeReq req); ListEdgesResp listEdges(1: ListEdgesReq req); - ListHostsResp listHosts(1: ListHostsReq req); + ExecResp addHosts(1: AddHostsReq req); + ExecResp addHostsIntoZone(1: AddHostsIntoZoneReq req); + ExecResp dropHosts(1: DropHostsReq req); + ListHostsResp listHosts(1: ListHostsReq req); GetPartsAllocResp getPartsAlloc(1: GetPartsAllocReq req); ListPartsResp listParts(1: ListPartsReq req); @@ -1182,17 +1198,17 @@ service MetaService { AdminJobResp runAdminJob(1: AdminJobReq req); - ExecResp addZone(1: AddZoneReq req); + ExecResp mergeZone(1: MergeZoneReq req); ExecResp dropZone(1: DropZoneReq req); - ExecResp addHostIntoZone(1: AddHostIntoZoneReq req); - ExecResp dropHostFromZone(1: DropHostFromZoneReq req); + ExecResp splitZone(1: SplitZoneReq req); + ExecResp renameZone(1: RenameZoneReq req); GetZoneResp getZone(1: GetZoneReq req); ListZonesResp listZones(1: ListZonesReq req); CreateBackupResp createBackup(1: CreateBackupReq req); - ExecResp restoreMeta(1: RestoreMetaReq req); - ExecResp addListener(1: AddListenerReq req); - ExecResp removeListener(1: RemoveListenerReq req); + ExecResp restoreMeta(1: RestoreMetaReq req); + ExecResp addListener(1: AddListenerReq req); + ExecResp removeListener(1: RemoveListenerReq req); ListListenerResp listListener(1: ListListenerReq req); GetStatsResp getStats(1: GetStatsReq req); diff --git a/src/meta/ActiveHostsMan.cpp b/src/meta/ActiveHostsMan.cpp index 05a435a1a47..c870c5127b8 100644 --- a/src/meta/ActiveHostsMan.cpp +++ b/src/meta/ActiveHostsMan.cpp @@ -85,13 +85,37 @@ nebula::cpp2::ErrorCode ActiveHostsMan::updateHostInfo(kvstore::KVStore* kv, return ret; } +bool ActiveHostsMan::machineRegisted(kvstore::KVStore* kv, const HostAddr& hostAddr) { + auto machineKey = MetaKeyUtils::machineKey(hostAddr.host, hostAddr.port); + std::string machineValue; + auto code = kv->get(kDefaultSpaceId, kDefaultPartId, machineKey, &machineValue); + return code == nebula::cpp2::ErrorCode::SUCCEEDED; +} + ErrorOr> ActiveHostsMan::getActiveHosts( kvstore::KVStore* kv, int32_t expiredTTL, cpp2::HostRole role) { + const auto& machinePrefix = MetaKeyUtils::machinePrefix(); + std::unique_ptr machineIter; + auto retCode = kv->prefix(kDefaultSpaceId, kDefaultPartId, machinePrefix, &machineIter); + if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Failed to get machines, error " << apache::thrift::util::enumNameSafe(retCode); + return retCode; + } + + std::vector machines; + while (machineIter->valid()) { + auto machine = MetaKeyUtils::parseMachineKey(machineIter->key()); + LOG(INFO) << "Machine " << machine; + machines.emplace_back(std::move(machine)); + machineIter->next(); + } + const auto& prefix = MetaKeyUtils::hostPrefix(); std::unique_ptr iter; - auto retCode = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + retCode = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Failed to getActiveHosts, error " << apache::thrift::util::enumNameSafe(retCode); + LOG(ERROR) << "Failed to get active hosts, error " + << apache::thrift::util::enumNameSafe(retCode); return retCode; } @@ -103,6 +127,14 @@ ErrorOr> ActiveHostsMan::getActiv while (iter->valid()) { auto host = MetaKeyUtils::parseHostKey(iter->key()); HostInfo info = HostInfo::decode(iter->val()); + + if (info.role_ == cpp2::HostRole::STORAGE && + std::find(machines.begin(), machines.end(), host) == machines.end()) { + retCode = nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND; + LOG(ERROR) << "Machine not found " << host; + break; + } + if (info.role_ == role) { if (now - info.lastHBTimeInMilliSec_ < threshold) { hosts.emplace_back(host.host, host.port); @@ -198,9 +230,18 @@ ErrorOr ActiveHostsMan::isLived(kvstore::KVStore* ErrorOr ActiveHostsMan::getHostInfo(kvstore::KVStore* kv, const HostAddr& host) { + auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); + std::string machineValue; + auto retCode = kv->get(kDefaultSpaceId, kDefaultPartId, machineKey, &machineValue); + if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Get machine info " << host + << " failed, error: " << apache::thrift::util::enumNameSafe(retCode); + return retCode; + } + auto hostKey = MetaKeyUtils::hostKey(host.host, host.port); std::string hostValue; - auto retCode = kv->get(kDefaultSpaceId, kDefaultPartId, hostKey, &hostValue); + retCode = kv->get(kDefaultSpaceId, kDefaultPartId, hostKey, &hostValue); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Get host info " << host << " failed, error: " << apache::thrift::util::enumNameSafe(retCode); diff --git a/src/meta/ActiveHostsMan.h b/src/meta/ActiveHostsMan.h index 4bc318559b2..97b70d35577 100644 --- a/src/meta/ActiveHostsMan.h +++ b/src/meta/ActiveHostsMan.h @@ -110,6 +110,8 @@ class ActiveHostsMan final { const HostInfo& info, const AllLeaders* leaderParts = nullptr); + static bool machineRegisted(kvstore::KVStore* kv, const HostAddr& hostAddr); + static ErrorOr> getActiveHosts( kvstore::KVStore* kv, int32_t expiredTTL = 0, cpp2::HostRole role = cpp2::HostRole::STORAGE); diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index 7faf784bc26..59737758909 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -80,11 +80,13 @@ nebula_add_library( processors/job/GetStatsProcessor.cpp processors/job/ListTagIndexStatusProcessor.cpp processors/job/ListEdgeIndexStatusProcessor.cpp - processors/zone/AddZoneProcessor.cpp + processors/zone/AddHostsProcessor.cpp + processors/zone/AddHostsIntoZoneProcessor.cpp + processors/zone/DropHostsProcessor.cpp processors/zone/DropZoneProcessor.cpp + processors/zone/RenameZoneProcessor.cpp processors/zone/GetZoneProcessor.cpp processors/zone/ListZonesProcessor.cpp - processors/zone/UpdateZoneProcessor.cpp processors/listener/ListenerProcessor.cpp processors/session/SessionManagerProcessor.cpp ) diff --git a/src/meta/MetaServiceHandler.cpp b/src/meta/MetaServiceHandler.cpp index 58a26b8c038..4eda958934a 100644 --- a/src/meta/MetaServiceHandler.cpp +++ b/src/meta/MetaServiceHandler.cpp @@ -61,11 +61,13 @@ #include "meta/processors/schema/ListTagsProcessor.h" #include "meta/processors/session/SessionManagerProcessor.h" #include "meta/processors/user/AuthenticationProcessor.h" -#include "meta/processors/zone/AddZoneProcessor.h" +#include "meta/processors/zone/AddHostsIntoZoneProcessor.h" +#include "meta/processors/zone/AddHostsProcessor.h" +#include "meta/processors/zone/DropHostsProcessor.h" #include "meta/processors/zone/DropZoneProcessor.h" #include "meta/processors/zone/GetZoneProcessor.h" #include "meta/processors/zone/ListZonesProcessor.h" -#include "meta/processors/zone/UpdateZoneProcessor.h" +#include "meta/processors/zone/RenameZoneProcessor.h" #define RETURN_FUTURE(processor) \ auto f = processor->getFuture(); \ @@ -116,6 +118,16 @@ folly::Future MetaServiceHandler::future_getSpace( RETURN_FUTURE(processor); } +folly::Future MetaServiceHandler::future_addHosts(const cpp2::AddHostsReq& req) { + auto* processor = AddHostsProcessor::instance(kvstore_); + RETURN_FUTURE(processor); +} + +folly::Future MetaServiceHandler::future_dropHosts(const cpp2::DropHostsReq& req) { + auto* processor = DropHostsProcessor::instance(kvstore_); + RETURN_FUTURE(processor); +} + folly::Future MetaServiceHandler::future_listHosts( const cpp2::ListHostsReq& req) { auto* processor = ListHostsProcessor::instance(kvstore_); @@ -417,13 +429,14 @@ folly::Future MetaServiceHandler::future_createBackup( RETURN_FUTURE(processor); } -folly::Future MetaServiceHandler::future_addZone(const cpp2::AddZoneReq& req) { - auto* processor = AddZoneProcessor::instance(kvstore_); +folly::Future MetaServiceHandler::future_dropZone(const cpp2::DropZoneReq& req) { + auto* processor = DropZoneProcessor::instance(kvstore_); RETURN_FUTURE(processor); } -folly::Future MetaServiceHandler::future_dropZone(const cpp2::DropZoneReq& req) { - auto* processor = DropZoneProcessor::instance(kvstore_); +folly::Future MetaServiceHandler::future_renameZone( + const cpp2::RenameZoneReq& req) { + auto* processor = RenameZoneProcessor::instance(kvstore_); RETURN_FUTURE(processor); } @@ -438,15 +451,9 @@ folly::Future MetaServiceHandler::future_listZones( RETURN_FUTURE(processor); } -folly::Future MetaServiceHandler::future_addHostIntoZone( - const cpp2::AddHostIntoZoneReq& req) { - auto* processor = AddHostIntoZoneProcessor::instance(kvstore_); - RETURN_FUTURE(processor); -} - -folly::Future MetaServiceHandler::future_dropHostFromZone( - const cpp2::DropHostFromZoneReq& req) { - auto* processor = DropHostFromZoneProcessor::instance(kvstore_); +folly::Future MetaServiceHandler::future_addHostsIntoZone( + const cpp2::AddHostsIntoZoneReq& req) { + auto* processor = AddHostsIntoZoneProcessor::instance(kvstore_); RETURN_FUTURE(processor); } @@ -533,5 +540,6 @@ folly::Future MetaServiceHandler::future_verifyCl auto* processor = VerifyClientVersionProcessor::instance(kvstore_); RETURN_FUTURE(processor); } + } // namespace meta } // namespace nebula diff --git a/src/meta/MetaServiceHandler.h b/src/meta/MetaServiceHandler.h index 1a66097c08c..87bb22bfa7b 100644 --- a/src/meta/MetaServiceHandler.h +++ b/src/meta/MetaServiceHandler.h @@ -38,6 +38,10 @@ class MetaServiceHandler final : public cpp2::MetaServiceSvIf { folly::Future future_getSpace(const cpp2::GetSpaceReq& req) override; + folly::Future future_addHosts(const cpp2::AddHostsReq& req) override; + + folly::Future future_dropHosts(const cpp2::DropHostsReq& req) override; + folly::Future future_listHosts(const cpp2::ListHostsReq& req) override; folly::Future future_listParts(const cpp2::ListPartsReq& req) override; @@ -176,19 +180,16 @@ class MetaServiceHandler final : public cpp2::MetaServiceSvIf { /** * Zone manager **/ - folly::Future future_addZone(const cpp2::AddZoneReq& req) override; - folly::Future future_dropZone(const cpp2::DropZoneReq& req) override; + folly::Future future_renameZone(const cpp2::RenameZoneReq& req) override; + folly::Future future_getZone(const cpp2::GetZoneReq& req) override; folly::Future future_listZones(const cpp2::ListZonesReq& req) override; - folly::Future future_addHostIntoZone( - const cpp2::AddHostIntoZoneReq& req) override; - - folly::Future future_dropHostFromZone( - const cpp2::DropHostFromZoneReq& req) override; + folly::Future future_addHostsIntoZone( + const cpp2::AddHostsIntoZoneReq& req) override; // listener folly::Future future_addListener(const cpp2::AddListenerReq& req) override; diff --git a/src/meta/processors/BaseProcessor-inl.h b/src/meta/processors/BaseProcessor-inl.h index dc9a2ec1bc2..04bcb80b232 100644 --- a/src/meta/processors/BaseProcessor-inl.h +++ b/src/meta/processors/BaseProcessor-inl.h @@ -261,14 +261,46 @@ nebula::cpp2::ErrorCode BaseProcessor::userExist(const std::string& accoun } template -nebula::cpp2::ErrorCode BaseProcessor::hostExist(const std::string& hostKey) { - auto ret = doGet(hostKey); +nebula::cpp2::ErrorCode BaseProcessor::machineExist(const std::string& machineKey) { + auto ret = doGet(machineKey); if (nebula::ok(ret)) { return nebula::cpp2::ErrorCode::SUCCEEDED; } return nebula::error(ret); } +template +nebula::cpp2::ErrorCode BaseProcessor::includeByZone(const std::vector& hosts) { + const auto& prefix = MetaKeyUtils::zonePrefix(); + auto iterRet = doPrefix(prefix); + nebula::cpp2::ErrorCode code = nebula::cpp2::ErrorCode::SUCCEEDED; + if (!nebula::ok(iterRet)) { + code = nebula::error(iterRet); + LOG(ERROR) << "Get zones failed, error: " << apache::thrift::util::enumNameSafe(code); + return code; + } + + auto iter = nebula::value(iterRet).get(); + while (iter->valid()) { + auto name = MetaKeyUtils::parseZoneName(iter->key()); + auto zoneHosts = MetaKeyUtils::parseZoneHosts(iter->val()); + for (const auto& host : hosts) { + if (std::find(zoneHosts.begin(), zoneHosts.end(), host) != zoneHosts.end()) { + LOG(ERROR) << "Host overlap found in zone " << name; + code = nebula::cpp2::ErrorCode::E_CONFLICT; + break; + } + } + + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + break; + } + iter->next(); + } + + return code; +} + template ErrorOr BaseProcessor::getSpaceId( const std::string& name) { @@ -561,21 +593,6 @@ bool BaseProcessor::checkIndexExist(const std::vector return true; } -template -ErrorOr BaseProcessor::getGroupId( - const std::string& groupName) { - auto indexKey = MetaKeyUtils::indexGroupKey(groupName); - auto ret = doGet(std::move(indexKey)); - if (nebula::ok(ret)) { - return *reinterpret_cast(nebula::value(ret).c_str()); - } - auto retCode = nebula::error(ret); - if (retCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { - retCode = nebula::cpp2::ErrorCode::E_GROUP_NOT_FOUND; - } - return retCode; -} - template ErrorOr BaseProcessor::getZoneId( const std::string& zoneName) { diff --git a/src/meta/processors/BaseProcessor.h b/src/meta/processors/BaseProcessor.h index 537a583d448..d09f4d73327 100644 --- a/src/meta/processors/BaseProcessor.h +++ b/src/meta/processors/BaseProcessor.h @@ -180,9 +180,14 @@ class BaseProcessor { nebula::cpp2::ErrorCode userExist(const std::string& account); /** - * Check host has been registered or not. + * Check machine has been registered or not. * */ - nebula::cpp2::ErrorCode hostExist(const std::string& hostKey); + nebula::cpp2::ErrorCode machineExist(const std::string& machineKey); + + /** + * Check hosts has been include by zone or not. + * */ + nebula::cpp2::ErrorCode includeByZone(const std::vector& hosts); /** * Return the spaceId for name. @@ -241,8 +246,6 @@ class BaseProcessor { bool checkIndexExist(const std::vector& fields, const cpp2::IndexItem& item); - ErrorOr getGroupId(const std::string& groupName); - ErrorOr getZoneId(const std::string& zoneName); nebula::cpp2::ErrorCode listenerExist(GraphSpaceID space, cpp2::ListenerType type); diff --git a/src/meta/processors/Common.h b/src/meta/processors/Common.h index 843d5876b7a..ef160578cf6 100644 --- a/src/meta/processors/Common.h +++ b/src/meta/processors/Common.h @@ -37,6 +37,7 @@ class LockUtils { GENERATE_LOCK(zone); GENERATE_LOCK(listener); GENERATE_LOCK(session); + GENERATE_LOCK(machine); #undef GENERATE_LOCK }; diff --git a/src/meta/processors/admin/HBProcessor.cpp b/src/meta/processors/admin/HBProcessor.cpp index 04e806cb320..ef2e05c5e6c 100644 --- a/src/meta/processors/admin/HBProcessor.cpp +++ b/src/meta/processors/admin/HBProcessor.cpp @@ -29,10 +29,18 @@ void HBProcessor::onFinished() { void HBProcessor::process(const cpp2::HBReq& req) { HostAddr host((*req.host_ref()).host, (*req.host_ref()).port); nebula::cpp2::ErrorCode ret; - + auto role = req.get_role(); LOG(INFO) << "Receive heartbeat from " << host - << ", role = " << apache::thrift::util::enumNameSafe(req.get_role()); - if (req.get_role() == cpp2::HostRole::STORAGE) { + << ", role = " << apache::thrift::util::enumNameSafe(role); + + if (role == cpp2::HostRole::STORAGE) { + if (!ActiveHostsMan::machineRegisted(kvstore_, host)) { + LOG(ERROR) << "Machine " << host << " is not registed"; + handleErrorCode(nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND); + onFinished(); + return; + } + ClusterID peerClusterId = req.get_cluster_id(); if (peerClusterId == 0) { LOG(INFO) << "Set clusterId for new host " << host << "!"; @@ -66,7 +74,7 @@ void HBProcessor::process(const cpp2::HBReq& req) { } } - HostInfo info(time::WallClock::fastNowInMilliSec(), req.get_role(), req.get_git_info_sha()); + HostInfo info(time::WallClock::fastNowInMilliSec(), role, req.get_git_info_sha()); if (req.leader_partIds_ref().has_value()) { ret = ActiveHostsMan::updateHostInfo(kvstore_, host, info, &*req.leader_partIds_ref()); } else { diff --git a/src/meta/processors/parts/CreateSpaceProcessor.cpp b/src/meta/processors/parts/CreateSpaceProcessor.cpp index c04da56cf88..9a4bd792c33 100644 --- a/src/meta/processors/parts/CreateSpaceProcessor.cpp +++ b/src/meta/processors/parts/CreateSpaceProcessor.cpp @@ -238,7 +238,7 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { if ((int32_t)hosts.size() < replicaFactor) { LOG(ERROR) << "Not enough hosts existed for replica " << replicaFactor << ", hosts num " << hosts.size(); - handleErrorCode(nebula::cpp2::ErrorCode::E_UNSUPPORTED); + handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); onFinished(); return; } diff --git a/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp b/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp new file mode 100644 index 00000000000..3e9a0e4281c --- /dev/null +++ b/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp @@ -0,0 +1,106 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "meta/processors/zone/AddHostsIntoZoneProcessor.h" + +DECLARE_int32(heartbeat_interval_secs); + +namespace nebula { +namespace meta { + +void AddHostsIntoZoneProcessor::process(const cpp2::AddHostsIntoZoneReq& req) { + folly::SharedMutex::ReadHolder zHolder(LockUtils::zoneLock()); + folly::SharedMutex::ReadHolder mHolder(LockUtils::machineLock()); + auto hosts = req.get_hosts(); + + // Confirm that there are no duplicates in the parameters. + if (std::unique(hosts.begin(), hosts.end()) != hosts.end()) { + LOG(ERROR) << "Hosts have duplicated element"; + handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); + onFinished(); + return; + } + + // Confirm that the parameter is not empty. + if (hosts.empty()) { + LOG(ERROR) << "Hosts is empty"; + handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); + onFinished(); + return; + } + + auto zoneName = req.get_zone_name(); + auto isNew = req.get_is_new(); + nebula::cpp2::ErrorCode code = nebula::cpp2::ErrorCode::SUCCEEDED; + std::vector data; + for (auto& host : hosts) { + // Ensure that the node is not registered. + auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); + if (machineExist(machineKey) == nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "The host " << host << " have existed!"; + code = nebula::cpp2::ErrorCode::E_EXISTED; + break; + } + data.emplace_back(MetaKeyUtils::machineKey(host.host, host.port), ""); + } + + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code); + onFinished(); + return; + } + + auto zoneKey = MetaKeyUtils::zoneKey(zoneName); + auto zoneValueRet = doGet(std::move(zoneKey)); + std::vector zoneHosts; + if (isNew) { + // If you are creating a new zone, should make sure the zone not existed. + if (nebula::ok(zoneValueRet)) { + LOG(ERROR) << "Zone " << zoneName + << " have existed error: " << apache::thrift::util::enumNameSafe(code); + handleErrorCode(nebula::cpp2::ErrorCode::E_EXISTED); + onFinished(); + return; + } + } else { + // If you are appending host into a zone, should make sure the zone have existed. + if (!nebula::ok(zoneValueRet)) { + code = nebula::error(zoneValueRet); + if (code == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + code = nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND; + } + LOG(ERROR) << "Get zone " << zoneName << " failed, error " + << apache::thrift::util::enumNameSafe(code); + handleErrorCode(code); + onFinished(); + return; + } + + auto originalHosts = MetaKeyUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); + zoneHosts.insert(zoneHosts.end(), originalHosts.begin(), originalHosts.end()); + } + + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code); + onFinished(); + return; + } + + // Check the hosts not exist in all zones + code = includeByZone(hosts); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code); + onFinished(); + return; + } + + zoneHosts.insert(zoneHosts.end(), hosts.begin(), hosts.end()); + data.emplace_back(std::move(zoneKey), MetaKeyUtils::zoneVal(std::move(zoneHosts))); + LOG(INFO) << "Add Hosts Into Zone " << zoneName; + doSyncPutAndUpdate(std::move(data)); +} + +} // namespace meta +} // namespace nebula diff --git a/src/meta/processors/zone/AddHostsIntoZoneProcessor.h b/src/meta/processors/zone/AddHostsIntoZoneProcessor.h new file mode 100644 index 00000000000..93929601f4b --- /dev/null +++ b/src/meta/processors/zone/AddHostsIntoZoneProcessor.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef META_ADDHOSTSINTOZONEPROCESSOR_H +#define META_ADDHOSTSINTOZONEPROCESSOR_H + +#include "meta/processors/BaseProcessor.h" + +namespace nebula { +namespace meta { + +class AddHostsIntoZoneProcessor : public BaseProcessor { + public: + static AddHostsIntoZoneProcessor* instance(kvstore::KVStore* kvstore) { + return new AddHostsIntoZoneProcessor(kvstore); + } + + void process(const cpp2::AddHostsIntoZoneReq& req); + + private: + explicit AddHostsIntoZoneProcessor(kvstore::KVStore* kvstore) + : BaseProcessor(kvstore) {} +}; + +} // namespace meta +} // namespace nebula +#endif // META_ADDHOSTSINTOZONEPROCESSOR_H diff --git a/src/meta/processors/zone/AddHostsProcessor.cpp b/src/meta/processors/zone/AddHostsProcessor.cpp new file mode 100644 index 00000000000..56afe5d9b3a --- /dev/null +++ b/src/meta/processors/zone/AddHostsProcessor.cpp @@ -0,0 +1,70 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "meta/processors/zone/AddHostsProcessor.h" + +#include "version/Version.h" + +namespace nebula { +namespace meta { + +void AddHostsProcessor::process(const cpp2::AddHostsReq& req) { + folly::SharedMutex::ReadHolder zHolder(LockUtils::zoneLock()); + folly::SharedMutex::ReadHolder mHolder(LockUtils::machineLock()); + auto hosts = req.get_hosts(); + // Confirm that there are no duplicates in the parameters. + if (std::unique(hosts.begin(), hosts.end()) != hosts.end()) { + LOG(ERROR) << "Hosts have duplicated element"; + handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); + onFinished(); + return; + } + + // Confirm that the parameter is not empty. + if (hosts.empty()) { + LOG(ERROR) << "Hosts is empty"; + handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); + onFinished(); + return; + } + + std::vector data; + nebula::cpp2::ErrorCode code; + for (auto& host : hosts) { + // Ensure that the node is not registered. + auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); + if (machineExist(machineKey) == nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "The host " << host << " have existed!"; + code = nebula::cpp2::ErrorCode::E_EXISTED; + break; + } + + // Automatic generation zone + auto zoneName = folly::stringPrintf("default_zone_%s_%d", host.host.c_str(), host.port); + auto zoneIdRet = getZoneId(zoneName); + if (nebula::ok(zoneIdRet)) { + LOG(ERROR) << "Zone " << zoneName << " have existed"; + code = nebula::error(zoneIdRet); + break; + } + + auto zoneKey = MetaKeyUtils::zoneKey(zoneName); + auto zoneVal = MetaKeyUtils::zoneVal({host}); + data.emplace_back(std::move(machineKey), ""); + data.emplace_back(std::move(zoneKey), std::move(zoneVal)); + } + + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code); + onFinished(); + return; + } + + doPut(std::move(data)); +} + +} // namespace meta +} // namespace nebula diff --git a/src/meta/processors/zone/AddHostsProcessor.h b/src/meta/processors/zone/AddHostsProcessor.h new file mode 100644 index 00000000000..5315aa6d5b1 --- /dev/null +++ b/src/meta/processors/zone/AddHostsProcessor.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#ifndef META_ADDHOSTSPROCESSOR_H +#define META_ADDHOSTSPROCESSOR_H + +#include "meta/processors/BaseProcessor.h" + +namespace nebula { +namespace meta { + +class AddHostsProcessor : public BaseProcessor { + public: + static AddHostsProcessor* instance(kvstore::KVStore* kvstore) { + return new AddHostsProcessor(kvstore); + } + + void process(const cpp2::AddHostsReq& req); + + private: + explicit AddHostsProcessor(kvstore::KVStore* kvstore) : BaseProcessor(kvstore) {} +}; + +} // namespace meta +} // namespace nebula + +#endif // META_ADDHOSTSPROCESSOR_H diff --git a/src/meta/processors/zone/AddZoneProcessor.cpp b/src/meta/processors/zone/AddZoneProcessor.cpp deleted file mode 100644 index 991f2570e81..00000000000 --- a/src/meta/processors/zone/AddZoneProcessor.cpp +++ /dev/null @@ -1,119 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "meta/processors/zone/AddZoneProcessor.h" - -DECLARE_int32(heartbeat_interval_secs); - -namespace nebula { -namespace meta { - -void AddZoneProcessor::process(const cpp2::AddZoneReq& req) { - folly::SharedMutex::WriteHolder wHolder(LockUtils::zoneLock()); - auto zoneName = req.get_zone_name(); - auto nodes = req.get_nodes(); - if (nodes.empty()) { - LOG(ERROR) << "The hosts should not be empty"; - handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); - onFinished(); - return; - } - - std::set nodeSet(nodes.begin(), nodes.end()); - if (nodes.size() != nodeSet.size()) { - LOG(ERROR) << "Conflict host found in the zone"; - handleErrorCode(nebula::cpp2::ErrorCode::E_CONFLICT); - onFinished(); - return; - } - - auto activeHostsRet = ActiveHostsMan::getActiveHosts(kvstore_); - if (!nebula::ok(activeHostsRet)) { - auto retCode = nebula::error(activeHostsRet); - LOG(ERROR) << "Create zone failed, error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - auto activeHosts = nebula::value(activeHostsRet); - - std::sort(activeHosts.begin(), activeHosts.end()); - if (!std::includes(activeHosts.begin(), activeHosts.end(), nodeSet.begin(), nodeSet.end())) { - LOG(ERROR) << "Host not exist"; - handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); - onFinished(); - return; - } - - auto zoneIdRet = getZoneId(zoneName); - if (nebula::ok(zoneIdRet)) { - LOG(ERROR) << "Zone " << zoneName << " already existed"; - handleErrorCode(nebula::cpp2::ErrorCode::E_EXISTED); - onFinished(); - return; - } else { - auto retCode = nebula::error(zoneIdRet); - if (retCode != nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND) { - LOG(ERROR) << "Create zone failed, zone name " << zoneName - << " error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - } - - // Check the node is not include in another zone - auto retCode = checkHostNotOverlap(nodes); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(retCode); - onFinished(); - return; - } - - auto zoneRet = autoIncrementId(); - if (!nebula::ok(zoneRet)) { - LOG(ERROR) << "Create Zone failed"; - handleErrorCode(nebula::error(zoneRet)); - onFinished(); - return; - } - - auto zoneId = nebula::value(zoneRet); - std::vector data; - data.emplace_back(MetaKeyUtils::indexZoneKey(zoneName), - std::string(reinterpret_cast(&zoneId), sizeof(ZoneID))); - data.emplace_back(MetaKeyUtils::zoneKey(zoneName), MetaKeyUtils::zoneVal(nodes)); - - LOG(INFO) << "Create Zone: " << zoneName; - doSyncPutAndUpdate(std::move(data)); -} - -nebula::cpp2::ErrorCode AddZoneProcessor::checkHostNotOverlap(const std::vector& nodes) { - const auto& prefix = MetaKeyUtils::zonePrefix(); - auto iterRet = doPrefix(prefix); - if (!nebula::ok(iterRet)) { - auto retCode = nebula::error(iterRet); - LOG(ERROR) << "Get zones failed, error: " << apache::thrift::util::enumNameSafe(retCode); - return retCode; - } - - auto iter = nebula::value(iterRet).get(); - while (iter->valid()) { - auto zoneName = MetaKeyUtils::parseZoneName(iter->key()); - auto hosts = MetaKeyUtils::parseZoneHosts(iter->val()); - for (auto& node : nodes) { - auto hostIter = std::find(hosts.begin(), hosts.end(), node); - if (hostIter != hosts.end()) { - LOG(ERROR) << "Host overlap found in zone " << zoneName; - return nebula::cpp2::ErrorCode::E_INVALID_PARM; - } - } - iter->next(); - } - return nebula::cpp2::ErrorCode::SUCCEEDED; -} - -} // namespace meta -} // namespace nebula diff --git a/src/meta/processors/zone/AddZoneProcessor.h b/src/meta/processors/zone/AddZoneProcessor.h deleted file mode 100644 index a9fcef47f76..00000000000 --- a/src/meta/processors/zone/AddZoneProcessor.h +++ /dev/null @@ -1,30 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef META_ADDZONEPROCESSOR_H -#define META_ADDZONEPROCESSOR_H - -#include "meta/processors/BaseProcessor.h" - -namespace nebula { -namespace meta { - -class AddZoneProcessor : public BaseProcessor { - public: - static AddZoneProcessor* instance(kvstore::KVStore* kvstore) { - return new AddZoneProcessor(kvstore); - } - - void process(const cpp2::AddZoneReq& req); - - private: - explicit AddZoneProcessor(kvstore::KVStore* kvstore) : BaseProcessor(kvstore) {} - - nebula::cpp2::ErrorCode checkHostNotOverlap(const std::vector& nodes); -}; - -} // namespace meta -} // namespace nebula -#endif // META_ADDZONEPROCESSOR_H diff --git a/src/meta/processors/zone/DropHostsProcessor.cpp b/src/meta/processors/zone/DropHostsProcessor.cpp new file mode 100644 index 00000000000..a3541cccf59 --- /dev/null +++ b/src/meta/processors/zone/DropHostsProcessor.cpp @@ -0,0 +1,157 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "meta/processors/zone/DropHostsProcessor.h" + +namespace nebula { +namespace meta { + +void DropHostsProcessor::process(const cpp2::DropHostsReq& req) { + auto hosts = req.get_hosts(); + if (std::unique(hosts.begin(), hosts.end()) != hosts.end()) { + LOG(ERROR) << "Hosts have duplicated element"; + handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); + onFinished(); + return; + } + + if (hosts.empty()) { + LOG(ERROR) << "Hosts is empty"; + handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); + onFinished(); + return; + } + + std::vector data; + // Check that partition is not held on the host + const auto& spacePrefix = MetaKeyUtils::spacePrefix(); + auto spaceIterRet = doPrefix(spacePrefix); + auto spaceIter = nebula::value(spaceIterRet).get(); + nebula::cpp2::ErrorCode code = nebula::cpp2::ErrorCode::SUCCEEDED; + while (spaceIter->valid()) { + auto spaceId = MetaKeyUtils::spaceId(spaceIter->key()); + auto spaceKey = MetaKeyUtils::spaceKey(spaceId); + auto ret = doGet(spaceKey); + if (!nebula::ok(ret)) { + code = nebula::error(ret); + LOG(ERROR) << "Get Space " << spaceId + << " error: " << apache::thrift::util::enumNameSafe(code); + break; + } + + const auto& partPrefix = MetaKeyUtils::partPrefix(spaceId); + auto partIterRet = doPrefix(partPrefix); + auto partIter = nebula::value(partIterRet).get(); + while (partIter->valid()) { + auto partHosts = MetaKeyUtils::parsePartVal(partIter->val()); + for (auto& h : partHosts) { + if (std::find(hosts.begin(), hosts.end(), h) != hosts.end()) { + LOG(ERROR) << h << " is related with partition"; + code = nebula::cpp2::ErrorCode::E_CONFLICT; + break; + } + } + partIter->next(); + } + spaceIter->next(); + } + + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code); + onFinished(); + return; + } + + // Remove points related to the zone + const auto& prefix = MetaKeyUtils::zonePrefix(); + auto iterRet = doPrefix(prefix); + if (!nebula::ok(iterRet)) { + auto retCode = nebula::error(iterRet); + LOG(ERROR) << "List zones failed, error: " << apache::thrift::util::enumNameSafe(retCode); + handleErrorCode(retCode); + onFinished(); + return; + } + + auto iter = nebula::value(iterRet).get(); + while (iter->valid()) { + auto hs = MetaKeyUtils::parseZoneHosts(iter->val()); + if (std::all_of(hs.begin(), hs.end(), [&hosts](auto& h) { + return std::find(hosts.begin(), hosts.end(), h) != hosts.end(); + })) { + auto zoneName = MetaKeyUtils::parseZoneName(iter->key()); + LOG(INFO) << "Drop zone " << zoneName; + code = checkRelatedSpace(zoneName); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + break; + } + data.emplace_back(iter->key()); + } + iter->next(); + } + + // Detach the machine from cluster + for (auto& host : hosts) { + auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); + auto ret = machineExist(machineKey); + if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "The host " << HostAddr(host.host, host.port) << " not existed!"; + code = nebula::cpp2::ErrorCode::E_NO_HOSTS; + break; + } + data.emplace_back(std::move(machineKey)); + } + + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code); + onFinished(); + return; + } + + resp_.set_code(nebula::cpp2::ErrorCode::SUCCEEDED); + doMultiRemove(std::move(data)); +} + +nebula::cpp2::ErrorCode DropHostsProcessor::checkRelatedSpace(const std::string& zoneName) { + const auto& prefix = MetaKeyUtils::spacePrefix(); + auto ret = doPrefix(prefix); + if (!nebula::ok(ret)) { + auto retCode = nebula::error(ret); + LOG(ERROR) << "List spaces failed, error " << apache::thrift::util::enumNameSafe(retCode); + return nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND; + } + + auto iter = nebula::value(ret).get(); + while (iter->valid()) { + auto properties = MetaKeyUtils::parseSpace(iter->val()); + size_t replicaFactor = properties.get_replica_factor(); + auto zones = properties.get_zone_names(); + LOG(INFO) << "replica_factor " << replicaFactor << " zone size " << zones.size(); + auto it = std::find(zones.begin(), zones.end(), zoneName); + if (it != zones.end()) { + if (zones.size() == replicaFactor) { + LOG(ERROR) << "Zone size is same with replica factor"; + return nebula::cpp2::ErrorCode::E_CONFLICT; + } else { + zones.erase(it); + properties.set_zone_names(zones); + rewriteSpaceProperties(iter->key().data(), properties); + } + } + iter->next(); + } + return nebula::cpp2::ErrorCode::SUCCEEDED; +} + +void DropHostsProcessor::rewriteSpaceProperties(const std::string& spaceKey, + const meta::cpp2::SpaceDesc& properties) { + auto spaceVal = MetaKeyUtils::spaceVal(properties); + std::vector data = {{spaceKey, spaceVal}}; + doSyncPutAndUpdate(std::move(data)); +} + +} // namespace meta +} // namespace nebula diff --git a/src/meta/processors/zone/DropHostsProcessor.h b/src/meta/processors/zone/DropHostsProcessor.h new file mode 100644 index 00000000000..a0bec3b27fd --- /dev/null +++ b/src/meta/processors/zone/DropHostsProcessor.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#ifndef META_DROPHOSTSPROCESSOR_H +#define META_DROPHOSTSPROCESSOR_H + +#include "meta/processors/BaseProcessor.h" + +namespace nebula { +namespace meta { + +class DropHostsProcessor : public BaseProcessor { + public: + static DropHostsProcessor* instance(kvstore::KVStore* kvstore) { + return new DropHostsProcessor(kvstore); + } + + void process(const cpp2::DropHostsReq& req); + + private: + explicit DropHostsProcessor(kvstore::KVStore* kvstore) : BaseProcessor(kvstore) {} + + nebula::cpp2::ErrorCode checkRelatedSpace(const std::string& zoneName); + + void rewriteSpaceProperties(const std::string& spaceKey, const meta::cpp2::SpaceDesc& properties); +}; + +} // namespace meta +} // namespace nebula + +#endif // META_DROPHOSTSPROCESSOR_H diff --git a/src/meta/processors/zone/DropZoneProcessor.cpp b/src/meta/processors/zone/DropZoneProcessor.cpp index a6377879b93..777dc939c22 100644 --- a/src/meta/processors/zone/DropZoneProcessor.cpp +++ b/src/meta/processors/zone/DropZoneProcessor.cpp @@ -24,14 +24,6 @@ void DropZoneProcessor::process(const cpp2::DropZoneReq& req) { return; } - // If zone belong to any group, it should not be dropped. - auto retCode = checkGroupDependency(zoneName); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(retCode); - onFinished(); - return; - } - std::vector keys; keys.emplace_back(MetaKeyUtils::indexZoneKey(zoneName)); keys.emplace_back(MetaKeyUtils::zoneKey(zoneName)); @@ -39,28 +31,5 @@ void DropZoneProcessor::process(const cpp2::DropZoneReq& req) { doSyncMultiRemoveAndUpdate(std::move(keys)); } -nebula::cpp2::ErrorCode DropZoneProcessor::checkGroupDependency(const std::string& zoneName) { - const auto& prefix = MetaKeyUtils::groupPrefix(); - auto iterRet = doPrefix(prefix); - if (!nebula::ok(iterRet)) { - auto retCode = nebula::error(iterRet); - LOG(ERROR) << "List zones failed, error: " << apache::thrift::util::enumNameSafe(retCode); - return retCode; - } - auto iter = nebula::value(iterRet).get(); - - while (iter->valid()) { - auto zoneNames = MetaKeyUtils::parseZoneNames(iter->val()); - auto zoneIter = std::find(zoneNames.begin(), zoneNames.end(), zoneName); - if (zoneIter != zoneNames.end()) { - auto groupName = MetaKeyUtils::parseGroupName(iter->key()); - LOG(ERROR) << "Zone " << zoneName << " is belong to Group " << groupName; - return nebula::cpp2::ErrorCode::E_NOT_DROP; - } - iter->next(); - } - return nebula::cpp2::ErrorCode::SUCCEEDED; -} - } // namespace meta } // namespace nebula diff --git a/src/meta/processors/zone/DropZoneProcessor.h b/src/meta/processors/zone/DropZoneProcessor.h index 5cbe6ef7c66..e231203533c 100644 --- a/src/meta/processors/zone/DropZoneProcessor.h +++ b/src/meta/processors/zone/DropZoneProcessor.h @@ -21,8 +21,6 @@ class DropZoneProcessor : public BaseProcessor { private: explicit DropZoneProcessor(kvstore::KVStore* kvstore) : BaseProcessor(kvstore) {} - - nebula::cpp2::ErrorCode checkGroupDependency(const std::string& zoneName); }; } // namespace meta diff --git a/src/meta/processors/zone/RenameZoneProcessor.cpp b/src/meta/processors/zone/RenameZoneProcessor.cpp new file mode 100644 index 00000000000..4dcbf8fe5b5 --- /dev/null +++ b/src/meta/processors/zone/RenameZoneProcessor.cpp @@ -0,0 +1,83 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "meta/processors/zone/RenameZoneProcessor.h" + +namespace nebula { +namespace meta { + +void RenameZoneProcessor::process(const cpp2::RenameZoneReq& req) { + auto originalZoneName = req.get_original_zone_name(); + auto originalZoneKey = MetaKeyUtils::zoneKey(originalZoneName); + auto originalZoneValueRet = doGet(std::move(originalZoneKey)); + if (!nebula::ok(originalZoneValueRet)) { + LOG(ERROR) << "Zone " << originalZoneName << " not existed"; + handleErrorCode(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND); + onFinished(); + return; + } + + auto originalZoneValue = nebula::value(originalZoneValueRet); + auto zoneName = req.get_zone_name(); + auto zoneKey = MetaKeyUtils::zoneKey(zoneName); + auto zoneValueRet = doGet(std::move(zoneKey)); + if (nebula::ok(zoneValueRet)) { + LOG(ERROR) << "Zone " << zoneName << " have existed"; + handleErrorCode(nebula::cpp2::ErrorCode::E_EXISTED); + onFinished(); + return; + } + + const auto& prefix = MetaKeyUtils::spacePrefix(); + auto ret = doPrefix(prefix); + if (!nebula::ok(ret)) { + LOG(ERROR) << "List spaces failed"; + handleErrorCode(nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND); + onFinished(); + return; + } + + std::vector data; + auto iter = nebula::value(ret).get(); + while (iter->valid()) { + auto spaceKey = iter->key(); + auto properties = MetaKeyUtils::parseSpace(iter->val()); + auto zones = properties.get_zone_names(); + auto it = std::find(zones.begin(), zones.end(), originalZoneName); + if (it != zones.end()) { + std::replace(zones.begin(), zones.end(), originalZoneName, zoneName); + properties.set_zone_names(zones); + auto spaceVal = MetaKeyUtils::spaceVal(properties); + data.emplace_back(spaceKey, std::move(spaceVal)); + } + iter->next(); + } + + folly::Baton baton; + auto result = nebula::cpp2::ErrorCode::SUCCEEDED; + std::vector keys = {originalZoneKey}; + kvstore_->asyncMultiRemove(kDefaultSpaceId, + kDefaultPartId, + std::move(keys), + [&result, &baton](nebula::cpp2::ErrorCode code) { + if (nebula::cpp2::ErrorCode::SUCCEEDED != code) { + result = code; + LOG(INFO) << "Remove data error on meta server"; + } + baton.post(); + }); + baton.wait(); + if (result != nebula::cpp2::ErrorCode::SUCCEEDED) { + this->handleErrorCode(result); + this->onFinished(); + return; + } + + data.emplace_back(zoneKey, originalZoneValue); + doSyncPutAndUpdate(std::move(data)); +} + +} // namespace meta +} // namespace nebula diff --git a/src/meta/processors/zone/RenameZoneProcessor.h b/src/meta/processors/zone/RenameZoneProcessor.h new file mode 100644 index 00000000000..f74033acaf3 --- /dev/null +++ b/src/meta/processors/zone/RenameZoneProcessor.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef META_RENAMEZONEPROCESSOR_H +#define META_RENAMEZONEPROCESSOR_H + +#include "meta/processors/BaseProcessor.h" + +namespace nebula { +namespace meta { + +class RenameZoneProcessor : public BaseProcessor { + public: + static RenameZoneProcessor* instance(kvstore::KVStore* kvstore) { + return new RenameZoneProcessor(kvstore); + } + + void process(const cpp2::RenameZoneReq& req); + + private: + explicit RenameZoneProcessor(kvstore::KVStore* kvstore) + : BaseProcessor(kvstore) {} +}; + +} // namespace meta +} // namespace nebula + +#endif // META_RENAMEZONEPROCESSOR_H diff --git a/src/meta/processors/zone/UpdateZoneProcessor.cpp b/src/meta/processors/zone/UpdateZoneProcessor.cpp deleted file mode 100644 index cb7af515804..00000000000 --- a/src/meta/processors/zone/UpdateZoneProcessor.cpp +++ /dev/null @@ -1,188 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "meta/processors/zone/UpdateZoneProcessor.h" - -DECLARE_int32(heartbeat_interval_secs); - -namespace nebula { -namespace meta { - -void AddHostIntoZoneProcessor::process(const cpp2::AddHostIntoZoneReq& req) { - folly::SharedMutex::ReadHolder rHolder(LockUtils::zoneLock()); - auto zoneName = req.get_zone_name(); - auto zoneIdRet = getZoneId(zoneName); - if (!nebula::ok(zoneIdRet)) { - auto retCode = nebula::error(zoneIdRet); - LOG(ERROR) << "Get Zone failed, zone " << zoneName - << " error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - - auto zoneKey = MetaKeyUtils::zoneKey(zoneName); - auto zoneValueRet = doGet(std::move(zoneKey)); - if (!nebula::ok(zoneValueRet)) { - auto retCode = nebula::error(zoneValueRet); - if (retCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { - retCode = nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND; - } - LOG(ERROR) << "Get zone " << zoneName << " failed, error " - << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - - auto hosts = MetaKeyUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); - auto host = req.get_node(); - // check this host not exist in all zones - const auto& prefix = MetaKeyUtils::zonePrefix(); - auto iterRet = doPrefix(prefix); - if (!nebula::ok(iterRet)) { - auto retCode = nebula::error(iterRet); - LOG(ERROR) << "Get zones failed, error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - - auto iter = nebula::value(iterRet).get(); - while (iter->valid()) { - auto name = MetaKeyUtils::parseZoneName(iter->key()); - auto zoneHosts = MetaKeyUtils::parseZoneHosts(iter->val()); - auto hostIter = std::find(zoneHosts.begin(), zoneHosts.end(), host); - if (hostIter != zoneHosts.end()) { - LOG(ERROR) << "Host overlap found in zone " << name; - handleErrorCode(nebula::cpp2::ErrorCode::E_EXISTED); - onFinished(); - return; - } - iter->next(); - } - - auto activeHostsRet = ActiveHostsMan::getActiveHosts(kvstore_); - if (!nebula::ok(activeHostsRet)) { - auto retCode = nebula::error(activeHostsRet); - LOG(ERROR) << "Get hosts failed, error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - auto activeHosts = nebula::value(activeHostsRet); - - auto found = std::find(activeHosts.begin(), activeHosts.end(), host); - if (found == activeHosts.end()) { - LOG(ERROR) << "Host " << host << " not exist"; - handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); - onFinished(); - return; - } - - hosts.emplace_back(host); - std::vector data; - data.emplace_back(std::move(zoneKey), MetaKeyUtils::zoneVal(std::move(hosts))); - LOG(INFO) << "Add Host " << host << " Into Zone " << zoneName; - doSyncPutAndUpdate(std::move(data)); -} - -void DropHostFromZoneProcessor::process(const cpp2::DropHostFromZoneReq& req) { - folly::SharedMutex::ReadHolder rHolder(LockUtils::zoneLock()); - auto zoneName = req.get_zone_name(); - auto zoneIdRet = getZoneId(zoneName); - if (!nebula::ok(zoneIdRet)) { - auto retCode = nebula::error(zoneIdRet); - LOG(ERROR) << "Get Zone failed, group " << zoneName - << " error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - - auto zoneKey = MetaKeyUtils::zoneKey(zoneName); - auto zoneValueRet = doGet(std::move(zoneKey)); - if (!nebula::ok(zoneValueRet)) { - auto retCode = nebula::error(zoneValueRet); - if (retCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { - retCode = nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND; - } - LOG(ERROR) << "Get zone " << zoneName - << " failed, error: " << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; - } - - const auto& spacePrefix = MetaKeyUtils::spacePrefix(); - auto spaceIterRet = doPrefix(spacePrefix); - auto spaceIter = nebula::value(spaceIterRet).get(); - auto code = nebula::cpp2::ErrorCode::SUCCEEDED; - while (spaceIter->valid()) { - auto spaceId = MetaKeyUtils::spaceId(spaceIter->key()); - auto spaceKey = MetaKeyUtils::spaceKey(spaceId); - auto ret = doGet(spaceKey); - if (!nebula::ok(ret)) { - code = nebula::error(ret); - LOG(ERROR) << "Get Space " << spaceId - << " error: " << apache::thrift::util::enumNameSafe(code); - break; - } - - auto properties = MetaKeyUtils::parseSpace(nebula::value(ret)); - - const auto& partPrefix = MetaKeyUtils::partPrefix(spaceId); - auto partIterRet = doPrefix(partPrefix); - auto partIter = nebula::value(partIterRet).get(); - - while (partIter->valid()) { - auto partHosts = MetaKeyUtils::parsePartVal(partIter->val()); - for (auto& h : partHosts) { - if (h == req.get_node()) { - LOG(ERROR) << h << " is related with partition"; - code = nebula::cpp2::ErrorCode::E_CONFLICT; - break; - } - } - - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - break; - } else { - partIter->next(); - } - } - - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - break; - } else { - spaceIter->next(); - } - } - - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(nebula::cpp2::ErrorCode::E_CONFLICT); - onFinished(); - return; - } - - auto hosts = MetaKeyUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); - auto host = req.get_node(); - auto iter = std::find(hosts.begin(), hosts.end(), host); - if (iter == hosts.end()) { - LOG(ERROR) << "Host " << host << " not exist in the zone " << zoneName; - handleErrorCode(nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND); - onFinished(); - return; - } - - hosts.erase(iter); - std::vector data; - data.emplace_back(std::move(zoneKey), MetaKeyUtils::zoneVal(std::move(hosts))); - LOG(INFO) << "Drop Host " << host << " From Zone " << zoneName; - doSyncPutAndUpdate(std::move(data)); -} - -} // namespace meta -} // namespace nebula diff --git a/src/meta/processors/zone/UpdateZoneProcessor.h b/src/meta/processors/zone/UpdateZoneProcessor.h deleted file mode 100644 index 2ddcb87d96b..00000000000 --- a/src/meta/processors/zone/UpdateZoneProcessor.h +++ /dev/null @@ -1,42 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef META_UPDATEZONEPROCESSOR_H -#define META_UPDATEZONEPROCESSOR_H - -#include "meta/processors/BaseProcessor.h" - -namespace nebula { -namespace meta { - -class AddHostIntoZoneProcessor : public BaseProcessor { - public: - static AddHostIntoZoneProcessor* instance(kvstore::KVStore* kvstore) { - return new AddHostIntoZoneProcessor(kvstore); - } - - void process(const cpp2::AddHostIntoZoneReq& req); - - private: - explicit AddHostIntoZoneProcessor(kvstore::KVStore* kvstore) - : BaseProcessor(kvstore) {} -}; - -class DropHostFromZoneProcessor : public BaseProcessor { - public: - static DropHostFromZoneProcessor* instance(kvstore::KVStore* kvstore) { - return new DropHostFromZoneProcessor(kvstore); - } - - void process(const cpp2::DropHostFromZoneReq& req); - - private: - explicit DropHostFromZoneProcessor(kvstore::KVStore* kvstore) - : BaseProcessor(kvstore) {} -}; - -} // namespace meta -} // namespace nebula -#endif // META_UPDATEZONEPROCESSOR_H diff --git a/src/meta/test/ActiveHostsManTest.cpp b/src/meta/test/ActiveHostsManTest.cpp index 7b3cd4ce38d..b88eea278b4 100644 --- a/src/meta/test/ActiveHostsManTest.cpp +++ b/src/meta/test/ActiveHostsManTest.cpp @@ -8,6 +8,7 @@ #include "common/base/Base.h" #include "common/fs/TempDir.h" +#include "common/utils/MetaKeyUtils.h" #include "meta/ActiveHostsMan.h" #include "meta/test/TestUtils.h" @@ -20,7 +21,7 @@ namespace meta { TEST(ActiveHostsManTest, EncodeDecodeHostInfoV2) { auto now = time::WallClock::fastNowInMilliSec(); auto role = cpp2::HostRole::STORAGE; - std::string strGitInfoSHA = gitInfoSha(); + auto strGitInfoSHA = gitInfoSha(); { HostInfo hostInfo(now, role, strGitInfoSHA); auto encodeHostInfo = HostInfo::encodeV2(hostInfo); @@ -52,6 +53,15 @@ TEST(ActiveHostsManTest, NormalTest) { fs::TempDir rootPath("/tmp/ActiveHostsManTest.XXXXXX"); FLAGS_heartbeat_interval_secs = 1; std::unique_ptr kv(MockCluster::initMetaKV(rootPath.path())); + + std::vector data; + data.emplace_back(nebula::MetaKeyUtils::machineKey("0", 0), ""); + data.emplace_back(nebula::MetaKeyUtils::machineKey("0", 1), ""); + data.emplace_back(nebula::MetaKeyUtils::machineKey("0", 2), ""); + folly::Baton baton; + kv->asyncMultiPut(kDefaultSpaceId, kDefaultPartId, std::move(data), [&](auto) { baton.post(); }); + baton.wait(); + auto now = time::WallClock::fastNowInMilliSec(); HostInfo info1(now, cpp2::HostRole::STORAGE, gitInfoSha()); ActiveHostsMan::updateHostInfo(kv.get(), HostAddr("0", 0), info1); @@ -98,8 +108,16 @@ TEST(ActiveHostsManTest, LeaderTest) { fs::TempDir rootPath("/tmp/ActiveHostsManTest.XXXXXX"); FLAGS_heartbeat_interval_secs = 1; std::unique_ptr kv(MockCluster::initMetaKV(rootPath.path())); - auto now = time::WallClock::fastNowInMilliSec(); + std::vector data; + data.emplace_back(nebula::MetaKeyUtils::machineKey("0", 0), ""); + data.emplace_back(nebula::MetaKeyUtils::machineKey("0", 1), ""); + data.emplace_back(nebula::MetaKeyUtils::machineKey("0", 2), ""); + folly::Baton baton; + kv->asyncMultiPut(kDefaultSpaceId, kDefaultPartId, std::move(data), [&](auto) { baton.post(); }); + baton.wait(); + + auto now = time::WallClock::fastNowInMilliSec(); HostInfo hInfo1(now, cpp2::HostRole::STORAGE, gitInfoSha()); HostInfo hInfo2(now + 2000, cpp2::HostRole::STORAGE, gitInfoSha()); ActiveHostsMan::updateHostInfo(kv.get(), HostAddr("0", 0), hInfo1); diff --git a/src/meta/test/AdminClientTest.cpp b/src/meta/test/AdminClientTest.cpp index bb98cb22c96..3a4ac07c134 100644 --- a/src/meta/test/AdminClientTest.cpp +++ b/src/meta/test/AdminClientTest.cpp @@ -308,6 +308,7 @@ TEST(AdminClientTest, SnapshotTest) { auto now = time::WallClock::fastNowInMilliSec(); HostAddr host(localIp, rpcServer->port_); HostAddr storageHost = Utils::getStoreAddrFromAdminAddr(host); + TestUtils::createSomeHosts(kv.get(), {storageHost}); ActiveHostsMan::updateHostInfo( kv.get(), storageHost, HostInfo(now, meta::cpp2::HostRole::STORAGE, "")); auto hostsRet = ActiveHostsMan::getActiveHosts(kv.get()); diff --git a/src/meta/test/BalancerTest.cpp b/src/meta/test/BalancerTest.cpp index 308e7db25d8..3ddbc6a5544 100644 --- a/src/meta/test/BalancerTest.cpp +++ b/src/meta/test/BalancerTest.cpp @@ -39,7 +39,7 @@ TEST(BalanceTest, BalanceTaskTest) { auto* kv = dynamic_cast(store.get()); HostAddr src("0", 0); HostAddr dst("1", 1); - TestUtils::registerHB(kv, {src, dst}); + TestUtils::createSomeHosts(kv, {src, dst}); DefaultValue>::SetFactory( [] { return folly::Future(Status::OK()); }); @@ -159,8 +159,7 @@ TEST(BalanceTest, SimpleTestWithZone) { {"zone_1", {{"1", 1}}}, {"zone_2", {{"2", 2}}}, {"zone_3", {{"3", 3}}}}; - GroupInfo groupInfo = {{"group_0", {"zone_0", "zone_1", "zone_2", "zone_3"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); + TestUtils::assembleZone(kv, zoneInfo); } { cpp2::SpaceDesc properties; @@ -218,8 +217,7 @@ TEST(BalanceTest, ExpansionZoneTest) { // create zone and group ZoneInfo zoneInfo = {{"zone_0", {{"0", 0}}}, {"zone_1", {{"1", 1}}}, {"zone_2", {{"2", 2}}}}; - GroupInfo groupInfo = {{"default_group", {"zone_0", "zone_1", "zone_2"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); + TestUtils::assembleZone(kv, zoneInfo); } { cpp2::SpaceDesc properties; @@ -257,8 +255,7 @@ TEST(BalanceTest, ExpansionZoneTest) { {"zone_1", {{"1", 1}}}, {"zone_2", {{"2", 2}}}, {"zone_3", {{"3", 3}}}}; - GroupInfo groupInfo = {{"default_group", {"zone_0", "zone_1", "zone_2", "zone_3"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); + TestUtils::assembleZone(kv, zoneInfo); } { cpp2::SpaceDesc properties; @@ -307,8 +304,7 @@ TEST(BalanceTest, ExpansionHostIntoZoneTest) { // create zone and group ZoneInfo zoneInfo = {{"zone_0", {{"0", 0}}}, {"zone_1", {{"1", 1}}}, {"zone_2", {{"2", 2}}}}; - GroupInfo groupInfo = {{"default_group", {"zone_0", "zone_1", "zone_2"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); + TestUtils::assembleZone(kv, zoneInfo); } { cpp2::SpaceDesc properties; @@ -345,8 +341,7 @@ TEST(BalanceTest, ExpansionHostIntoZoneTest) { ZoneInfo zoneInfo = {{"zone_0", {{"0", 0}, {"3", 3}}}, {"zone_1", {{"1", 1}, {"4", 4}}}, {"zone_2", {{"2", 2}, {"5", 5}}}}; - GroupInfo groupInfo = {{"default_group", {"zone_0", "zone_1", "zone_2"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); + TestUtils::assembleZone(kv, zoneInfo); } { HostParts hostParts; @@ -386,8 +381,7 @@ TEST(BalanceTest, ShrinkZoneTest) { {"zone_1", {{"1", 1}}}, {"zone_2", {{"2", 2}}}, {"zone_3", {{"3", 3}}}}; - GroupInfo groupInfo = {{"default_group", {"zone_0", "zone_1", "zone_2", "zone_3"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); + TestUtils::assembleZone(kv, zoneInfo); } { cpp2::SpaceDesc properties; @@ -437,8 +431,7 @@ TEST(BalanceTest, ShrinkHostFromZoneTest) { ZoneInfo zoneInfo = {{"zone_0", {{"0", 0}, {"3", 3}}}, {"zone_1", {{"1", 1}, {"4", 4}}}, {"zone_2", {{"2", 2}, {"5", 5}}}}; - GroupInfo groupInfo = {{"default_group", {"zone_0", "zone_1", "zone_2"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); + TestUtils::assembleZone(kv, zoneInfo); } { cpp2::SpaceDesc properties; @@ -470,8 +463,7 @@ TEST(BalanceTest, ShrinkHostFromZoneTest) { { ZoneInfo zoneInfo = { {"zone_0", {{"0", 0}}}, {"zone_1", {{"1", 1}, {"4", 4}}}, {"zone_2", {{"2", 2}, {"5", 5}}}}; - GroupInfo groupInfo = {{"default_group", {"zone_0", "zone_1", "zone_2"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); + TestUtils::assembleZone(kv, zoneInfo); } balancer.lostHosts_ = {{"3", 3}}; ret = balancer.executeInternal(HostAddr(), {}); @@ -502,23 +494,7 @@ TEST(BalanceTest, BalanceWithComplexZoneTest) { {"zone_7", {HostAddr("14", 14), HostAddr("15", 15)}}, {"zone_8", {HostAddr("16", 16), HostAddr("17", 17)}}, }; - { - GroupInfo groupInfo = {{"group_0", {"zone_0", "zone_1", "zone_2", "zone_3", "zone_4"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); - } - { - GroupInfo groupInfo = {{"group_1", - {"zone_0", - "zone_1", - "zone_2", - "zone_3", - "zone_4", - "zone_5", - "zone_6", - "zone_7", - "zone_8"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); - } + TestUtils::assembleZone(kv, zoneInfo); } { { @@ -888,6 +864,7 @@ TEST(BalanceTest, BalancePlanTest) { JobDescription jd( testJobId.fetch_add(1, std::memory_order_relaxed), cpp2::AdminCmd::DATA_BALANCE, {}); BalancePlan plan(jd, kv, &client); + TestUtils::createSomeHosts(kv, hosts); TestUtils::registerHB(kv, hosts); for (int i = 0; i < 10; i++) { @@ -1674,8 +1651,7 @@ TEST(BalanceTest, LeaderBalanceWithZoneTest) { ZoneInfo zoneInfo = {{"zone_0", {HostAddr("0", 0), HostAddr("1", 1)}}, {"zone_1", {HostAddr("2", 2), HostAddr("3", 3)}}, {"zone_2", {HostAddr("4", 4), HostAddr("5", 5)}}}; - GroupInfo groupInfo = {{"default_group", {"zone_0", "zone_1", "zone_2"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); + TestUtils::assembleZone(kv, zoneInfo); } { cpp2::SpaceDesc properties; @@ -1753,8 +1729,7 @@ TEST(BalanceTest, LeaderBalanceWithLargerZoneTest) { {"zone_3", {HostAddr("6", 6), HostAddr("7", 7)}}, {"zone_4", {HostAddr("8", 8), HostAddr("9", 9)}}, }; - GroupInfo groupInfo = {{"default_group", {"zone_0", "zone_1", "zone_2", "zone_3", "zone_4"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); + TestUtils::assembleZone(kv, zoneInfo); } { cpp2::SpaceDesc properties; @@ -1823,23 +1798,7 @@ TEST(BalanceTest, LeaderBalanceWithComplexZoneTest) { {"zone_7", {HostAddr("14", 14), HostAddr("15", 15)}}, {"zone_8", {HostAddr("16", 16), HostAddr("17", 17)}}, }; - { - GroupInfo groupInfo = {{"group_0", {"zone_0", "zone_1", "zone_2", "zone_3", "zone_4"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); - } - { - GroupInfo groupInfo = {{"group_1", - {"zone_0", - "zone_1", - "zone_2", - "zone_3", - "zone_4", - "zone_5", - "zone_6", - "zone_7", - "zone_8"}}}; - TestUtils::assembleGroupAndZone(kv, zoneInfo, groupInfo); - } + TestUtils::assembleZone(kv, zoneInfo); } { { diff --git a/src/meta/test/CMakeLists.txt b/src/meta/test/CMakeLists.txt index e8591bae05a..83c2851bff8 100644 --- a/src/meta/test/CMakeLists.txt +++ b/src/meta/test/CMakeLists.txt @@ -195,21 +195,6 @@ nebula_add_test( gmock ) -nebula_add_test( - NAME - group_zone_test - SOURCES - GroupZoneTest.cpp - OBJECTS - ${meta_test_deps} - LIBRARIES - ${ROCKSDB_LIBRARIES} - ${THRIFT_LIBRARIES} - ${PROXYGEN_LIBRARIES} - wangle - gtest -) - nebula_add_test( NAME create_backup_test diff --git a/src/meta/test/CreateBackupProcessorTest.cpp b/src/meta/test/CreateBackupProcessorTest.cpp index d942d37647f..b06341894d3 100644 --- a/src/meta/test/CreateBackupProcessorTest.cpp +++ b/src/meta/test/CreateBackupProcessorTest.cpp @@ -87,6 +87,14 @@ TEST(ProcessorTest, CreateBackupTest) { fs::TempDir rootPath("/tmp/create_backup_test.XXXXXX"); std::unique_ptr kv(MockCluster::initMetaKV(rootPath.path())); + + std::vector machines; + machines.emplace_back(nebula::MetaKeyUtils::machineKey(localIp, rpcServer->port_), ""); + + folly::Baton b; + kv->asyncMultiPut(kDefaultSpaceId, kDefaultPartId, std::move(machines), [&](auto) { b.post(); }); + b.wait(); + auto now = time::WallClock::fastNowInMilliSec(); HostAddr host(localIp, rpcServer->port_); ActiveHostsMan::updateHostInfo(kv.get(), host, HostInfo(now, meta::cpp2::HostRole::STORAGE, "")); diff --git a/src/meta/test/GroupZoneTest.cpp b/src/meta/test/GroupZoneTest.cpp deleted file mode 100644 index 564cb1f7955..00000000000 --- a/src/meta/test/GroupZoneTest.cpp +++ /dev/null @@ -1,382 +0,0 @@ -/* Copyright (c) 2018 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include - -#include "common/base/Base.h" -#include "common/fs/TempDir.h" -#include "meta/processors/parts/CreateSpaceProcessor.h" -#include "meta/processors/parts/DropSpaceProcessor.h" -#include "meta/processors/zone/AddZoneProcessor.h" -#include "meta/processors/zone/DropZoneProcessor.h" -#include "meta/processors/zone/GetZoneProcessor.h" -#include "meta/processors/zone/ListZonesProcessor.h" -#include "meta/processors/zone/UpdateZoneProcessor.h" -#include "meta/test/TestUtils.h" - -namespace nebula { -namespace meta { - -TEST(GroupAndZoneTest, GroupAndZoneTest) { - fs::TempDir rootPath("/tmp/GroupZoneTest.XXXXXX"); - - // Prepare - std::unique_ptr kv(MockCluster::initMetaKV(rootPath.path())); - std::vector addresses; - for (int32_t i = 0; i < 13; i++) { - addresses.emplace_back(std::to_string(i), i); - } - TestUtils::registerHB(kv.get(), addresses); - { - cpp2::ListHostsReq req; - req.set_type(cpp2::ListHostType::STORAGE); - auto* processor = ListHostsProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(13, (*resp.hosts_ref()).size()); - for (auto i = 0; i < 13; i++) { - ASSERT_EQ(std::to_string(i), (*resp.hosts_ref())[i].get_hostAddr().host); - ASSERT_EQ(i, (*resp.hosts_ref())[i].get_hostAddr().port); - ASSERT_EQ(cpp2::HostStatus::ONLINE, (*resp.hosts_ref())[i].get_status()); - } - } - // Add Zone - {{std::vector nodes; - for (int32_t i = 0; i < 3; i++) { - nodes.emplace_back(std::to_string(i), i); - } - cpp2::AddZoneReq req; - req.set_zone_name("zone_0"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -{ - std::vector nodes; - for (int32_t i = 3; i < 6; i++) { - nodes.emplace_back(std::to_string(i), i); - } - cpp2::AddZoneReq req; - req.set_zone_name("zone_1"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -{ - std::vector nodes; - for (int32_t i = 6; i < 9; i++) { - nodes.emplace_back(std::to_string(i), i); - } - cpp2::AddZoneReq req; - req.set_zone_name("zone_2"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Host have overlap -{ - std::vector nodes; - for (int32_t i = 8; i < 11; i++) { - nodes.emplace_back(std::to_string(i), i); - } - cpp2::AddZoneReq req; - req.set_zone_name("zone_3"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); -} -} // namespace meta -// Add Zone with empty node list -{ - std::vector nodes; - cpp2::AddZoneReq req; - req.set_zone_name("zone_0"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); -} -// Add Zone with duplicate node -{ - std::vector nodes; - nodes.emplace_back(std::to_string(1), 1); - nodes.emplace_back(std::to_string(1), 1); - - cpp2::AddZoneReq req; - req.set_zone_name("zone_0"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_CONFLICT, resp.get_code()); -} -// Add Zone which node not exist -{ - std::vector nodes = {{"zone_not_exist", 0}}; - cpp2::AddZoneReq req; - req.set_zone_name("zone_4"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); -} -// Add Zone already existed -{ - std::vector nodes; - for (int32_t i = 0; i < 3; i++) { - nodes.emplace_back(std::to_string(i), i); - } - - cpp2::AddZoneReq req; - req.set_zone_name("zone_0"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); -} -// Get Zone -{ - cpp2::GetZoneReq req; - req.set_zone_name("zone_0"); - auto* processor = GetZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - std::vector nodes = {{"0", 0}, {"1", 1}, {"2", 2}}; - ASSERT_EQ(nodes, *resp.hosts_ref()); -} -// Get Zone which is not exist -{ - cpp2::GetZoneReq req; - req.set_zone_name("zone_not_exist"); - auto* processor = GetZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND, resp.get_code()); -} -// List Zones -{ - cpp2::ListZonesReq req; - auto* processor = ListZonesProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - ASSERT_EQ(3, (*resp.zones_ref()).size()); - ASSERT_EQ("zone_0", (*resp.zones_ref())[0].get_zone_name()); - ASSERT_EQ("zone_1", (*resp.zones_ref())[1].get_zone_name()); - ASSERT_EQ("zone_2", (*resp.zones_ref())[2].get_zone_name()); -} -// Add host into zone -{ - cpp2::AddHostIntoZoneReq req; - req.set_zone_name("zone_0"); - HostAddr node{"12", 12}; - req.set_node(std::move(node)); - auto* processor = AddHostIntoZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Add host into zone overlap with another zone -{ - cpp2::AddHostIntoZoneReq req; - req.set_zone_name("zone_0"); - HostAddr node{"3", 3}; - req.set_node(std::move(node)); - auto* processor = AddHostIntoZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); -} -// Add host into zone which zone is not exist -{ - cpp2::AddHostIntoZoneReq req; - req.set_zone_name("zone_not_exist"); - HostAddr node{"4", 4}; - req.set_node(std::move(node)); - auto* processor = AddHostIntoZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND, resp.get_code()); -} -// Add host into zone which the node have existed -{ - cpp2::AddHostIntoZoneReq req; - req.set_zone_name("zone_0"); - HostAddr node{"0", 0}; - req.set_node(std::move(node)); - auto* processor = AddHostIntoZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); -} -// Add host into zone which the node not existed -{ - cpp2::AddHostIntoZoneReq req; - req.set_zone_name("zone_0"); - HostAddr node{"99", 99}; - req.set_node(std::move(node)); - auto* processor = AddHostIntoZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); -} -// Drop host from zone which zone is not exist -{ - cpp2::DropHostFromZoneReq req; - req.set_zone_name("zone_not_exist"); - HostAddr node{"3", 3}; - req.set_node(std::move(node)); - auto* processor = DropHostFromZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND, resp.get_code()); -} -// Drop host from zone which the node not exist -{ - cpp2::DropHostFromZoneReq req; - req.set_zone_name("zone_0"); - HostAddr node{"3", 3}; - req.set_node(std::move(node)); - auto* processor = DropHostFromZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND, resp.get_code()); -} -{ - cpp2::DropHostFromZoneReq req; - req.set_zone_name("zone_0"); - HostAddr node{"12", 12}; - req.set_node(std::move(node)); - auto* processor = DropHostFromZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Drop Zone -{ - cpp2::DropZoneReq req; - req.set_zone_name("zone_0"); - auto* processor = DropZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); -} -// Drop Zone which is not exist -{ - cpp2::DropZoneReq req; - req.set_zone_name("zone_0"); - auto* processor = DropZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND, resp.get_code()); -} -} // namespace nebula - -TEST(GroupAndZoneTest, DropHostAndZoneTest) { - fs::TempDir rootPath("/tmp/DropHostAndZoneTest.XXXXXX"); - - // Prepare - std::unique_ptr kv(MockCluster::initMetaKV(rootPath.path())); - std::vector addresses; - for (int32_t i = 0; i < 1; i++) { - addresses.emplace_back(std::to_string(i), i); - } - TestUtils::registerHB(kv.get(), addresses); - { - cpp2::ListHostsReq req; - req.set_type(cpp2::ListHostType::STORAGE); - auto* processor = ListHostsProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(1, (*resp.hosts_ref()).size()); - for (auto i = 0; i < 1; i++) { - ASSERT_EQ(std::to_string(i), (*resp.hosts_ref())[i].get_hostAddr().host); - ASSERT_EQ(i, (*resp.hosts_ref())[i].get_hostAddr().port); - ASSERT_EQ(cpp2::HostStatus::ONLINE, (*resp.hosts_ref())[i].get_status()); - } - } - - // Add Zone - { - std::vector nodes; - nodes.emplace_back("0", 0); - cpp2::AddZoneReq req; - req.set_zone_name("zone_0"); - req.set_nodes(std::move(nodes)); - auto* processor = AddZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - } - // Drop host from zone - { - cpp2::DropHostFromZoneReq req; - req.set_zone_name("zone_0"); - HostAddr node{"0", 0}; - req.set_node(std::move(node)); - auto* processor = DropHostFromZoneProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - } - // List Zones - { - cpp2::ListZonesReq req; - auto* processor = ListZonesProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - ASSERT_EQ(1, (*resp.zones_ref()).size()); - ASSERT_EQ("zone_0", (*resp.zones_ref())[0].get_zone_name()); - } -} - -} // namespace meta -} // namespace nebula - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - folly::init(&argc, &argv, true); - google::SetStderrLogging(google::INFO); - return RUN_ALL_TESTS(); -} diff --git a/src/meta/test/HBProcessorTest.cpp b/src/meta/test/HBProcessorTest.cpp index 1eea3696eb0..77a4d515e44 100644 --- a/src/meta/test/HBProcessorTest.cpp +++ b/src/meta/test/HBProcessorTest.cpp @@ -7,6 +7,7 @@ #include "common/base/Base.h" #include "common/fs/TempDir.h" +#include "common/utils/MetaKeyUtils.h" #include "meta/processors/admin/HBProcessor.h" #include "meta/test/TestUtils.h" @@ -16,6 +17,17 @@ namespace meta { TEST(HBProcessorTest, HBTest) { fs::TempDir rootPath("/tmp/HBTest.XXXXXX"); std::unique_ptr kv(MockCluster::initMetaKV(rootPath.path())); + + std::vector machines; + for (auto i = 0; i < 5; i++) { + machines.emplace_back(nebula::MetaKeyUtils::machineKey(std::to_string(i), i), ""); + } + + folly::Baton baton; + kv->asyncMultiPut( + kDefaultSpaceId, kDefaultPartId, std::move(machines), [&](auto) { baton.post(); }); + baton.wait(); + const ClusterID kClusterId = 10; { for (auto i = 0; i < 5; i++) { @@ -48,7 +60,7 @@ TEST(HBProcessorTest, HBTest) { auto f = processor->getFuture(); processor->process(req); auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_WRONGCLUSTER, resp.get_code()); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND, resp.get_code()); } } diff --git a/src/meta/test/MetaClientTest.cpp b/src/meta/test/MetaClientTest.cpp index c7b3aa93de2..dcf477d2e91 100644 --- a/src/meta/test/MetaClientTest.cpp +++ b/src/meta/test/MetaClientTest.cpp @@ -37,6 +37,7 @@ TEST(MetaClientTest, InterfacesTest) { mock::MockCluster cluster; cluster.startMeta(rootPath.path()); + auto options = meta::MetaClientOptions(); cluster.initMetaClient(); auto* kv = cluster.metaKV_.get(); auto* client = cluster.metaClient_.get(); @@ -340,78 +341,6 @@ TEST(MetaClientTest, InterfacesTest) { } } -TEST(MetaClientTest, SpaceWithGroupTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/SpaceWithGroupTest.XXXXXX"); - - mock::MockCluster cluster; - cluster.startMeta(rootPath.path()); - cluster.initMetaClient(); - auto* kv = cluster.metaKV_.get(); - auto* client = cluster.metaClient_.get(); - - // Prepare - {{// Add hosts automatically, then testing listHosts interface. - std::vector addresses; - for (int32_t i = 0; i < 10; i++) { - addresses.emplace_back(std::to_string(i), i); - } - TestUtils::registerHB(kv, addresses); - auto ret = client->listHosts().get(); - ASSERT_TRUE(ret.ok()); - for (auto i = 0u; i < addresses.size(); i++) { - auto tHost = ret.value()[i].get_hostAddr(); - auto hostAddr = HostAddr(tHost.host, tHost.port); - ASSERT_EQ(addresses[i], hostAddr); - } -} -// Add Zone -{ - std::vector nodes = {{"0", 0}, {"1", 1}}; - auto result = client->addZone("zone_0", nodes).get(); - ASSERT_TRUE(result.ok()); -} -{ - std::vector nodes = {{"2", 2}, {"3", 3}}; - auto result = client->addZone("zone_1", nodes).get(); - ASSERT_TRUE(result.ok()); -} -{ - std::vector nodes = {{"4", 4}, {"5", 5}}; - auto result = client->addZone("zone_2", nodes).get(); - ASSERT_TRUE(result.ok()); -} -{ - std::vector nodes = {{"6", 6}, {"7", 7}}; - auto result = client->addZone("zone_3", nodes).get(); - ASSERT_TRUE(result.ok()); -} -{ - std::vector nodes = {{"8", 8}, {"9", 9}}; - auto result = client->addZone("zone_4", nodes).get(); - ASSERT_TRUE(result.ok()); -} -// List Zones -{ - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - ASSERT_EQ(5, result.value().size()); -} -} // namespace meta -// Create Space without Group -{ - meta::cpp2::SpaceDesc spaceDesc; - spaceDesc.set_space_name("default_space"); - spaceDesc.set_partition_num(9); - spaceDesc.set_replica_factor(3); - auto ret = client->createSpace(spaceDesc).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); - - ret = client->createSpace(spaceDesc, true).get(); - ASSERT_TRUE(ret.ok()) << ret.status(); -} -} // namespace nebula - TEST(MetaClientTest, TagTest) { FLAGS_heartbeat_interval_secs = 1; fs::TempDir rootPath("/tmp/MetaClientTagTest.XXXXXX"); @@ -1103,143 +1032,6 @@ TEST(MetaClientTest, EdgeIndexTest) { } } -TEST(MetaClientTest, GroupAndZoneTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/GroupAndZoneTest.XXXXXX"); - - mock::MockCluster cluster; - cluster.startMeta(rootPath.path()); - cluster.initMetaClient(); - auto* kv = cluster.metaKV_.get(); - auto* client = cluster.metaClient_.get(); - std::vector hosts; - for (int32_t i = 0; i < 13; i++) { - hosts.emplace_back(std::to_string(i), i); - } - TestUtils::createSomeHosts(kv, std::move(hosts)); - - // Add Zone - { - std::vector nodes = {{"0", 0}, {"1", 1}, {"2", 2}}; - auto result = client->addZone("zone_0", nodes).get(); - ASSERT_TRUE(result.ok()); - } - { - std::vector nodes = {{"3", 3}, {"4", 4}, {"5", 5}}; - auto result = client->addZone("zone_1", nodes).get(); - ASSERT_TRUE(result.ok()); - } - { - std::vector nodes = {{"6", 6}, {"7", 7}, {"8", 8}}; - auto result = client->addZone("zone_2", nodes).get(); - ASSERT_TRUE(result.ok()); - } - // Host have overlap - { - std::vector nodes = {{"8", 8}, {"9", 9}, {"10", 10}}; - auto result = client->addZone("zone_3", nodes).get(); - ASSERT_FALSE(result.ok()); - } - // Add Zone with empty node list - { - std::vector nodes; - auto result = client->addZone("zone_0", nodes).get(); - ASSERT_FALSE(result.ok()); - } - // Add Zone with duplicate node - { - std::vector nodes = {{"0", 0}, {"0", 0}}; - auto result = client->addZone("zone_0", nodes).get(); - ASSERT_FALSE(result.ok()); - } - // Add Zone which node not exist - { - std::vector nodes = {{"zone_not_exist", 0}}; - auto result = client->addZone("zone_4", nodes).get(); - ASSERT_FALSE(result.ok()); - } - // Add Zone already existed - { - std::vector nodes = {{"0", 0}, {"1", 1}, {"2", 2}}; - auto result = client->addZone("zone_0", nodes).get(); - ASSERT_FALSE(result.ok()); - } - // Get Zone - { - auto result = client->getZone("zone_0").get(); - ASSERT_TRUE(result.ok()); - } - // Get Zone which is not exist - { - auto result = client->getZone("zone_not_exist").get(); - ASSERT_FALSE(result.ok()); - } - // List Zones - { - auto result = client->listZones().get(); - ASSERT_TRUE(result.ok()); - ASSERT_EQ(3, result.value().size()); - } - // Add host into zone - { - HostAddr node("12", 12); - auto result = client->addHostIntoZone(node, "zone_0").get(); - ASSERT_TRUE(result.ok()); - } - // Add host into zone overlap with another zone - { - HostAddr node("3", 3); - auto result = client->addHostIntoZone(node, "zone_0").get(); - ASSERT_FALSE(result.ok()); - } - // Add host into zone which zone is not exist - { - HostAddr node("4", 4); - auto result = client->addHostIntoZone(node, "zone_not_exist").get(); - ASSERT_FALSE(result.ok()); - } - // Add host into zone which the node have existed - { - HostAddr node("0", 0); - auto result = client->addHostIntoZone(node, "zone_0").get(); - ASSERT_FALSE(result.ok()); - } - // Add host into zone which the node not existed - { - HostAddr node("99", 99); - auto result = client->addHostIntoZone(node, "zone_0").get(); - ASSERT_FALSE(result.ok()); - } - // Drop host from zone - { - HostAddr node("12", 12); - auto result = client->dropHostFromZone(node, "zone_0").get(); - ASSERT_TRUE(result.ok()); - } - // Drop host from zone which zone is not exist - { - HostAddr node("4", 4); - auto result = client->dropHostFromZone(node, "zone_not_exist").get(); - ASSERT_FALSE(result.ok()); - } - // Drop host from zone which the node not exist - { - HostAddr node("4", 4); - auto result = client->dropHostFromZone(node, "zone_0").get(); - ASSERT_FALSE(result.ok()); - } - // Drop Zone - { - auto result = client->dropZone("zone_0").get(); - ASSERT_TRUE(result.ok()); - } - // Drop Zone which is not exist - { - auto result = client->dropZone("zone_0").get(); - ASSERT_FALSE(result.ok()); - } -} - TEST(MetaClientTest, FTServiceTest) { FLAGS_heartbeat_interval_secs = 1; fs::TempDir rootPath("/tmp/FTServiceTest.XXXXXX"); @@ -1380,18 +1172,18 @@ TEST(MetaClientTest, DiffTest) { mock::MockCluster cluster; cluster.startMeta(rootPath.path()); - meta::MetaClientOptions options; - options.role_ = meta::cpp2::HostRole::STORAGE; - cluster.initMetaClient(options); auto* kv = cluster.metaKV_.get(); + cluster.initMetaClient(); auto* client = cluster.metaClient_.get(); + std::vector hosts = {{"0", 0}}; + TestUtils::createSomeHosts(kv, std::move(hosts)); auto listener = std::make_unique(); client->registerListener(listener.get()); { // Add hosts automatically, then testing listHosts interface. - std::vector hosts = {{"", 0}}; - TestUtils::registerHB(kv, hosts); + std::vector hbHosts = {{"0", 0}}; + TestUtils::registerHB(kv, hbHosts); auto ret = client->listHosts().get(); ASSERT_TRUE(ret.ok()); for (auto i = 0u; i < hosts.size(); i++) { @@ -1439,17 +1231,17 @@ TEST(MetaClientTest, ListenerDiffTest) { mock::MockCluster cluster; cluster.startMeta(rootPath.path()); - meta::MetaClientOptions options; - options.localHost_ = {"", 0}; - options.role_ = meta::cpp2::HostRole::STORAGE; - cluster.initMetaClient(options); + cluster.initMetaClient(); auto* kv = cluster.metaKV_.get(); + std::vector hosts = {{"0", 0}}; + TestUtils::createSomeHosts(kv, std::move(hosts)); auto* console = cluster.metaClient_.get(); auto testListener = std::make_unique(); console->registerListener(testListener.get()); // create another meta client for listener host HostAddr listenerHost("listener", 0); + meta::MetaClientOptions options; options.localHost_ = listenerHost; options.role_ = meta::cpp2::HostRole::UNKNOWN; auto threadPool = std::make_shared(1); @@ -1461,8 +1253,8 @@ TEST(MetaClientTest, ListenerDiffTest) { client->registerListener(listener.get()); // register HB for storage - std::vector hosts = {{"", 0}}; - TestUtils::registerHB(kv, hosts); + std::vector hbHosts = {{"0", 0}}; + TestUtils::registerHB(kv, hbHosts); { // create two space meta::cpp2::SpaceDesc spaceDesc; @@ -1534,17 +1326,20 @@ TEST(MetaClientTest, ListenerDiffTest) { TEST(MetaClientTest, HeartbeatTest) { FLAGS_heartbeat_interval_secs = 1; - const nebula::ClusterID kClusterId = 10; + // const nebula::ClusterID kClusterId = 10; fs::TempDir rootPath("/tmp/HeartbeatTest.XXXXXX"); mock::MockCluster cluster; cluster.startMeta(rootPath.path()); + auto* kv = cluster.metaKV_.get(); - meta::MetaClientOptions options; - HostAddr localHost(cluster.localIP(), network::NetworkUtils::getAvailablePort()); - options.localHost_ = localHost; - options.clusterId_ = kClusterId; - options.role_ = meta::cpp2::HostRole::STORAGE; - cluster.initMetaClient(std::move(options)); + // meta::MetaClientOptions options; + TestUtils::createSomeHosts(kv, {{"0", 0}}); + + HostAddr localHost("0", 0); + // options.localHost_ = localHost; + // options.clusterId_ = kClusterId; + // options.role_ = meta::cpp2::HostRole::STORAGE; + cluster.initMetaClient(); auto* client = cluster.metaClient_.get(); auto listener = std::make_unique(); @@ -1637,7 +1432,7 @@ TEST(MetaClientTest, RetryWithExceptionTest) { auto threadPool = std::make_shared(1); auto clientPort = network::NetworkUtils::getAvailablePort(); HostAddr localHost{localIp, clientPort}; - auto client = std::make_shared(threadPool, std::vector{HostAddr("0", 0)}); + auto client = std::make_shared(threadPool, std::vector{{"0", 0}}); // Retry with exception, then failed { LOG(INFO) << "Test heart beat..."; @@ -1944,6 +1739,8 @@ TEST(MetaClientTest, RocksdbOptionsTest) { mock::MockCluster cluster; cluster.startMeta(rootPath.path(), HostAddr("127.0.0.1", 0)); + auto* kv = cluster.metaKV_.get(); + TestUtils::createSomeHosts(kv, {{"0", 0}}); MetaClientOptions options; // Now the `--local_config' option only affect if initialize the configuration @@ -2003,6 +1800,460 @@ TEST(MetaClientTest, RocksdbOptionsTest) { } } +TEST(MetaClientTest, HostsTest) { + FLAGS_heartbeat_interval_secs = 1; + fs::TempDir rootPath("/tmp/HostsTest.XXXXXX"); + mock::MockCluster cluster; + cluster.startMeta(rootPath.path(), HostAddr("127.0.0.1", 0)); + cluster.initMetaClient(); + auto* client = cluster.metaClient_.get(); + { + // Add single host + std::vector hosts = {{"127.0.0.1", 8989}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } + { + // Add multi host + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } + { + // Add duplicated hosts + std::vector hosts = {{"127.0.0.1", 8986}, {"127.0.0.1", 8986}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_FALSE(result.ok()); + } + { + // Add empty hosts + std::vector hosts = {}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_FALSE(result.ok()); + } + { + // Add hosts which is existed + std::vector hosts = {{"127.0.0.1", 8988}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_FALSE(result.ok()); + } + { + // Add hosts which is existed + std::vector hosts = {{"127.0.0.1", 8986}, {"127.0.0.1", 8988}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_FALSE(result.ok()); + } + { + // Show the zones created by add hosts + auto result = client->listZones().get(); + ASSERT_TRUE(result.ok()); + ASSERT_EQ(3, result.value().size()); + } + { + // Drop hosts with duplicate element + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8987}}; + auto result = client->dropHosts(std::move(hosts)).get(); + EXPECT_FALSE(result.ok()); + } + { + // Drop hosts which is empty + std::vector hosts = {}; + auto result = client->dropHosts(std::move(hosts)).get(); + EXPECT_FALSE(result.ok()); + } +} + +TEST(MetaClientTest, AddHostsIntoNewZoneTest) { + FLAGS_heartbeat_interval_secs = 1; + fs::TempDir rootPath("/tmp/AddHostsIntoNewZoneTest.XXXXXX"); + mock::MockCluster cluster; + cluster.startMeta(rootPath.path(), HostAddr("127.0.0.1", 0)); + cluster.initMetaClient(); + auto* client = cluster.metaClient_.get(); + { + // Add host into zone with duplicate hosts + std::vector hosts = {{"127.0.0.1", 8988}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); + EXPECT_FALSE(result.ok()); + } + { + // Add host into new zone with empty hosts. + std::vector hosts = {}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); + EXPECT_FALSE(result.ok()); + } + { + // Add host into new zone. + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); + EXPECT_TRUE(result.ok()); + } + { + // Add host into new zone with zone name conflict. + std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); + EXPECT_FALSE(result.ok()); + } + { + // the hosts have exist in another zones. + std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8989}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_1", true).get(); + EXPECT_FALSE(result.ok()); + } +} + +TEST(MetaClientTest, AddHostsIntoZoneTest) { + FLAGS_heartbeat_interval_secs = 1; + fs::TempDir rootPath("/tmp/AddHostsIntoZoneTest.XXXXXX"); + mock::MockCluster cluster; + cluster.startMeta(rootPath.path(), HostAddr("127.0.0.1", 0)); + cluster.initMetaClient(); + auto* client = cluster.metaClient_.get(); + { + // Add host into zone with duplicate hosts + std::vector hosts = {{"127.0.0.1", 8988}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", false).get(); + EXPECT_FALSE(result.ok()); + } + { + // Add host into zone with empty hosts + std::vector hosts; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", false).get(); + EXPECT_FALSE(result.ok()); + } + { + // Add host into zone which zone is not exist. + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_not_existed", false).get(); + EXPECT_FALSE(result.ok()); + } + { + // Add host into zone. + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); + EXPECT_TRUE(result.ok()); + } + { + // Add host into zone with zone name conflict. + std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); + EXPECT_FALSE(result.ok()); + } + { + // Add existed hosts + std::vector hosts = {{"127.0.0.1", 8988}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); + EXPECT_FALSE(result.ok()); + } + { + std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_1", true).get(); + EXPECT_TRUE(result.ok()); + } + { + // Add existed hosts. + std::vector hosts = {{"127.0.0.1", 8988}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_FALSE(result.ok()); + } + { + std::vector hosts = {{"127.0.0.1", 8976}, {"127.0.0.1", 8988}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_1", false).get(); + EXPECT_FALSE(result.ok()); + } + { + // Drop hosts which is empty. + std::vector hosts = {}; + auto result = client->dropHosts(std::move(hosts)).get(); + EXPECT_FALSE(result.ok()); + } + { + // Drop hosts which have duplicate element. + std::vector hosts = {{"127.0.0.1", 8988}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + auto result = client->dropHosts(std::move(hosts)).get(); + EXPECT_FALSE(result.ok()); + } + { + // Drop hosts. + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + auto result = client->dropHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } +} + +TEST(MetaClientTest, DropHostsTest) { + FLAGS_heartbeat_interval_secs = 1; + fs::TempDir rootPath("/tmp/DropHostsTest.XXXXXX"); + mock::MockCluster cluster; + cluster.startMeta(rootPath.path(), HostAddr("127.0.0.1", 0)); + cluster.initMetaClient(); + auto* client = cluster.metaClient_.get(); + auto* kv = cluster.metaKV_.get(); + { + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } + TestUtils::registerHB(kv, {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}); + { + auto result = client->listZones().get(); + ASSERT_TRUE(result.ok()); + auto zones = result.value(); + ASSERT_EQ(3, zones.size()); + ASSERT_EQ("default_zone_127.0.0.1_8987", zones[0].get_zone_name()); + ASSERT_EQ("default_zone_127.0.0.1_8988", zones[1].get_zone_name()); + ASSERT_EQ("default_zone_127.0.0.1_8989", zones[2].get_zone_name()); + } + { + // Create Space on cluster, the replica number same with the zone size. + meta::cpp2::SpaceDesc properties; + properties.set_space_name("default_space_0"); + properties.set_partition_num(9); + properties.set_replica_factor(3); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + auto ret = client->createSpace(properties).get(); + ASSERT_TRUE(ret.ok()) << ret.status(); + ASSERT_EQ(1, ret.value()); + } + { + // Create Space on cluster, the replica number less than the zone size. + cpp2::SpaceDesc properties; + properties.set_space_name("default_space_1"); + properties.set_partition_num(9); + properties.set_replica_factor(1); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + auto ret = client->createSpace(properties).get(); + ASSERT_TRUE(ret.ok()) << ret.status(); + ASSERT_EQ(2, ret.value()); + } + { + // Create Space on cluster, the replica number greater than the zone size. + cpp2::SpaceDesc properties; + properties.set_space_name("default_space_2"); + properties.set_partition_num(9); + properties.set_replica_factor(6); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + auto ret = client->createSpace(properties).get(); + ASSERT_FALSE(ret.ok()) << ret.status(); + } + { + std::vector hosts = {{"127.0.0.1", 8976}, {"127.0.0.1", 8977}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); + EXPECT_TRUE(result.ok()); + } + { + std::vector hosts = {{"127.0.0.1", 8978}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_1", true).get(); + EXPECT_TRUE(result.ok()); + } + { + std::vector hosts = {{"127.0.0.1", 8979}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_2", true).get(); + EXPECT_TRUE(result.ok()); + } + TestUtils::registerHB( + kv, {{"127.0.0.1", 8976}, {"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}); + { + auto result = client->listZones().get(); + ASSERT_TRUE(result.ok()); + auto zones = result.value(); + ASSERT_EQ(6, zones.size()); + ASSERT_EQ("default_zone_127.0.0.1_8987", zones[0].get_zone_name()); + ASSERT_EQ("default_zone_127.0.0.1_8988", zones[1].get_zone_name()); + ASSERT_EQ("default_zone_127.0.0.1_8989", zones[2].get_zone_name()); + ASSERT_EQ("zone_0", zones[3].get_zone_name()); + ASSERT_EQ("zone_1", zones[4].get_zone_name()); + ASSERT_EQ("zone_2", zones[5].get_zone_name()); + } + { + // Create Space on cluster, the replica number greater than the zone size. + cpp2::SpaceDesc properties; + properties.set_space_name("default_space_on_zone_3"); + properties.set_partition_num(9); + properties.set_replica_factor(3); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + std::vector zones = {"zone_0", "zone_1", "zone_2"}; + properties.set_zone_names(std::move(zones)); + auto ret = client->createSpace(properties).get(); + ASSERT_TRUE(ret.ok()) << ret.status(); + ASSERT_EQ(4, ret.value()); + } + { + // Create Space on cluster, the replica number less than the zone size + cpp2::SpaceDesc properties; + properties.set_space_name("default_space_on_zone_1"); + properties.set_partition_num(9); + properties.set_replica_factor(1); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + std::vector zones = {"zone_0"}; + properties.set_zone_names(std::move(zones)); + auto ret = client->createSpace(properties).get(); + ASSERT_TRUE(ret.ok()) << ret.status(); + ASSERT_EQ(5, ret.value()); + } + { + // Create Space on cluster, the replica number greater than the zone size + cpp2::SpaceDesc properties; + properties.set_space_name("default_space_on_zone_6"); + properties.set_partition_num(9); + properties.set_replica_factor(6); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + std::vector zones = {"zone_0"}; + properties.set_zone_names(std::move(zones)); + auto ret = client->createSpace(properties).get(); + ASSERT_FALSE(ret.ok()) << ret.status(); + } + { + // Drop hosts which hold partition. + std::vector hosts = {{"127.0.0.1", 8987}}; + auto result = client->dropHosts(std::move(hosts)).get(); + EXPECT_FALSE(result.ok()); + } + { + // Drop hosts which hold partition. + std::vector hosts = {{"127.0.0.1", 8977}}; + auto result = client->dropHosts(std::move(hosts)).get(); + EXPECT_FALSE(result.ok()); + } + { + auto ret = client->dropSpace("default_space_on_zone_1").get(); + ASSERT_TRUE(ret.ok()); + } + { + auto ret = client->dropSpace("default_space_on_zone_3").get(); + ASSERT_TRUE(ret.ok()); + } + { + auto ret = client->dropSpace("default_space_0").get(); + ASSERT_TRUE(ret.ok()); + } + { + auto ret = client->dropSpace("default_space_1").get(); + ASSERT_TRUE(ret.ok()); + } + { + std::vector hosts = {{"127.0.0.1", 8987}}; + auto result = client->dropHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } + { + std::vector hosts = {{"127.0.0.1", 8976}}; + auto result = client->dropHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } + { + std::vector hosts = {{"127.0.0.1", 8978}}; + auto result = client->dropHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } + { + auto result = client->listZones().get(); + ASSERT_TRUE(result.ok()); + auto zones = result.value(); + ASSERT_EQ(4, zones.size()); + ASSERT_EQ("default_zone_127.0.0.1_8988", zones[0].get_zone_name()); + ASSERT_EQ("default_zone_127.0.0.1_8989", zones[1].get_zone_name()); + ASSERT_EQ("zone_0", zones[2].get_zone_name()); + ASSERT_EQ("zone_2", zones[3].get_zone_name()); + } +} + +TEST(MetaClientTest, RenameZoneTest) { + FLAGS_heartbeat_interval_secs = 1; + fs::TempDir rootPath("/tmp/RenameZoneTest.XXXXXX"); + mock::MockCluster cluster; + cluster.startMeta(rootPath.path(), HostAddr("127.0.0.1", 0)); + cluster.initMetaClient(); + auto* client = cluster.metaClient_.get(); + auto* kv = cluster.metaKV_.get(); + { + std::vector hosts = {{"127.0.0.1", 8987}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_0", true).get(); + EXPECT_TRUE(result.ok()); + } + { + std::vector hosts = {{"127.0.0.1", 8988}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_1", true).get(); + EXPECT_TRUE(result.ok()); + } + { + std::vector hosts = {{"127.0.0.1", 8989}}; + auto result = client->addHostsIntoZone(std::move(hosts), "zone_2", true).get(); + EXPECT_TRUE(result.ok()); + } + TestUtils::registerHB(kv, {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}); + { + auto result = client->listZones().get(); + ASSERT_TRUE(result.ok()); + auto zones = result.value(); + ASSERT_EQ(3, zones.size()); + ASSERT_EQ("zone_0", zones[0].get_zone_name()); + ASSERT_EQ("zone_1", zones[1].get_zone_name()); + ASSERT_EQ("zone_2", zones[2].get_zone_name()); + } + { + cpp2::SpaceDesc properties; + properties.set_space_name("default"); + properties.set_partition_num(9); + properties.set_replica_factor(3); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + std::vector zones = {"zone_0", "zone_1", "zone_2"}; + properties.set_zone_names(std::move(zones)); + auto ret = client->createSpace(properties).get(); + ASSERT_TRUE(ret.ok()) << ret.status(); + ASSERT_EQ(1, ret.value()); + } + { + auto ret = client->getSpace("default").get(); + ASSERT_TRUE(ret.ok()) << ret.status(); + auto spaceDesc = ret.value().get_properties(); + ASSERT_EQ("default", spaceDesc.get_space_name()); + ASSERT_EQ(9, spaceDesc.get_partition_num()); + ASSERT_EQ(3, spaceDesc.get_replica_factor()); + ASSERT_EQ("utf8", spaceDesc.get_charset_name()); + ASSERT_EQ("utf8_bin", spaceDesc.get_collate_name()); + auto zones = spaceDesc.get_zone_names(); + ASSERT_EQ(3, zones.size()); + ASSERT_EQ("zone_0", zones[0]); + ASSERT_EQ("zone_1", zones[1]); + ASSERT_EQ("zone_2", zones[2]); + } + { + auto result = client->renameZone("zone_not_exist", "new_zone_name").get(); + EXPECT_FALSE(result.ok()); + } + { + auto result = client->renameZone("zone_0", "zone_1").get(); + EXPECT_FALSE(result.ok()); + } + { + auto result = client->renameZone("zone_1", "z_1").get(); + EXPECT_TRUE(result.ok()); + } + { + auto ret = client->getSpace("default").get(); + ASSERT_TRUE(ret.ok()) << ret.status(); + auto spaceDesc = ret.value().get_properties(); + ASSERT_EQ("default", spaceDesc.get_space_name()); + ASSERT_EQ(9, spaceDesc.get_partition_num()); + ASSERT_EQ(3, spaceDesc.get_replica_factor()); + ASSERT_EQ("utf8", spaceDesc.get_charset_name()); + ASSERT_EQ("utf8_bin", spaceDesc.get_collate_name()); + auto zones = spaceDesc.get_zone_names(); + ASSERT_EQ(3, zones.size()); + ASSERT_EQ("zone_0", zones[0]); + ASSERT_EQ("z_1", zones[1]); + ASSERT_EQ("zone_2", zones[2]); + } +} + } // namespace meta } // namespace nebula diff --git a/src/meta/test/ProcessorTest.cpp b/src/meta/test/ProcessorTest.cpp index f42f5d4a5d9..f9e2662378d 100644 --- a/src/meta/test/ProcessorTest.cpp +++ b/src/meta/test/ProcessorTest.cpp @@ -30,9 +30,11 @@ #include "meta/processors/schema/ListEdgesProcessor.h" #include "meta/processors/schema/ListTagsProcessor.h" #include "meta/processors/session/SessionManagerProcessor.h" -#include "meta/processors/zone/AddZoneProcessor.h" +#include "meta/processors/zone/AddHostsIntoZoneProcessor.h" +#include "meta/processors/zone/AddHostsProcessor.h" +#include "meta/processors/zone/DropHostsProcessor.h" #include "meta/processors/zone/ListZonesProcessor.h" -#include "meta/processors/zone/UpdateZoneProcessor.h" +#include "meta/processors/zone/RenameZoneProcessor.h" #include "meta/test/TestUtils.h" DECLARE_int32(expired_threshold_sec); @@ -343,7 +345,6 @@ TEST(ProcessorTest, SpaceTest) { ASSERT_EQ("utf8", resp.get_item().get_properties().get_charset_name()); ASSERT_EQ("utf8_bin", resp.get_item().get_properties().get_collate_name()); } - { cpp2::ListSpacesReq req; auto* processor = ListSpacesProcessor::instance(kv.get()); @@ -2626,6 +2627,960 @@ TEST(ProcessorTest, TagIdAndEdgeTypeInSpaceRangeTest) { } } +TEST(ProcessorTest, HostsTest) { + fs::TempDir rootPath("/tmp/HostsTest.XXXXXX"); + auto kv = MockCluster::initMetaKV(rootPath.path()); + { + // Attempt to register heartbeat + const ClusterID kClusterId = 10; + for (auto i = 8987; i < 8990; i++) { + cpp2::HBReq req; + req.set_role(cpp2::HostRole::STORAGE); + req.set_host(HostAddr("127.0.0.1", i)); + req.set_cluster_id(kClusterId); + req.set_role(cpp2::HostRole::STORAGE); + auto* processor = HBProcessor::instance(kv.get(), nullptr, kClusterId); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND, resp.get_code()); + } + } + { + // Add single host + cpp2::AddHostsReq req; + std::vector hosts; + hosts.emplace_back("127.0.0.1", 8989); + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Add multi host + cpp2::AddHostsReq req; + std::vector hosts; + hosts.emplace_back("127.0.0.1", 8987); + hosts.emplace_back("127.0.0.1", 8988); + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Attempt to register heartbeat + const ClusterID kClusterId = 10; + for (auto i = 8987; i < 8990; i++) { + cpp2::HBReq req; + req.set_role(cpp2::HostRole::STORAGE); + req.set_host(HostAddr("127.0.0.1", i)); + req.set_cluster_id(kClusterId); + req.set_role(cpp2::HostRole::STORAGE); + auto* processor = HBProcessor::instance(kv.get(), nullptr, kClusterId); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + } + { + // Add duplicated hosts + cpp2::AddHostsReq req; + std::vector hosts; + hosts.emplace_back("127.0.0.1", 8986); + hosts.emplace_back("127.0.0.1", 8986); + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } + { + // Add empty hosts + cpp2::AddHostsReq req; + std::vector hosts; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } + { + // Add hosts which is existed + cpp2::AddHostsReq req; + std::vector hosts; + hosts.emplace_back("127.0.0.1", 8988); + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); + } + { + // Add hosts which is existed + cpp2::AddHostsReq req; + std::vector hosts; + hosts.emplace_back("127.0.0.1", 8986); + hosts.emplace_back("127.0.0.1", 8988); + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); + } + { + // Show the zones created by add hosts + cpp2::ListZonesReq req; + auto* processor = ListZonesProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + ASSERT_EQ(3, resp.get_zones().size()); + ASSERT_EQ("default_zone_127.0.0.1_8987", resp.get_zones()[0].get_zone_name()); + ASSERT_EQ("default_zone_127.0.0.1_8988", resp.get_zones()[1].get_zone_name()); + ASSERT_EQ("default_zone_127.0.0.1_8989", resp.get_zones()[2].get_zone_name()); + } + { + // Drop hosts with duplicate element + cpp2::DropHostsReq req; + std::vector hosts; + hosts.emplace_back("127.0.0.1", 8987); + hosts.emplace_back("127.0.0.1", 8987); + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } + { + // Drop hosts which is empty + cpp2::DropHostsReq req; + std::vector hosts; + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } + { + // Drop hosts which is not exist + cpp2::DropHostsReq req; + std::vector hosts; + hosts.emplace_back("127.0.0.1", 8986); + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_NO_HOSTS, resp.get_code()); + } + { + // Drop hosts successful + cpp2::DropHostsReq req; + std::vector hosts; + hosts.emplace_back("127.0.0.1", 8987); + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Drop hosts successful + cpp2::DropHostsReq req; + std::vector hosts; + hosts.emplace_back("127.0.0.1", 8988); + hosts.emplace_back("127.0.0.1", 8989); + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Attempt to register heartbeat + const ClusterID kClusterId = 10; + for (auto i = 8987; i < 8990; i++) { + cpp2::HBReq req; + req.set_role(cpp2::HostRole::STORAGE); + req.set_host(HostAddr("127.0.0.1", i)); + req.set_cluster_id(kClusterId); + req.set_role(cpp2::HostRole::STORAGE); + auto* processor = HBProcessor::instance(kv.get(), nullptr, kClusterId); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND, resp.get_code()); + } + } +} + +TEST(ProcessorTest, AddHostsIntoNewZoneTest) { + fs::TempDir rootPath("/tmp/AddHostsIntoZoneTest.XXXXXX"); + auto kv = MockCluster::initMetaKV(rootPath.path()); + { + // Attempt to register heartbeat + const ClusterID kClusterId = 10; + for (auto i = 8987; i < 8990; i++) { + cpp2::HBReq req; + req.set_role(cpp2::HostRole::STORAGE); + req.set_host(HostAddr("127.0.0.1", i)); + req.set_cluster_id(kClusterId); + req.set_role(cpp2::HostRole::STORAGE); + auto* processor = HBProcessor::instance(kv.get(), nullptr, kClusterId); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND, resp.get_code()); + } + } + { + // Add host into new zone with duplicate hosts. + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_0"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8988}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } + { + // Add host into new zone with empty hosts. + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_0"); + req.set_is_new(true); + req.set_hosts({}); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } + { + // Add host into new zone. + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_0"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Attempt to register heartbeat + const ClusterID kClusterId = 10; + for (auto i = 8987; i < 8990; i++) { + cpp2::HBReq req; + req.set_role(cpp2::HostRole::STORAGE); + req.set_host(HostAddr("127.0.0.1", i)); + req.set_cluster_id(kClusterId); + req.set_role(cpp2::HostRole::STORAGE); + auto* processor = HBProcessor::instance(kv.get(), nullptr, kClusterId); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + } + { + // Add host into new zone with zone name conflict. + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_0"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); + } + { + // the hosts have exist in another zones. + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_1"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8989}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); + } +} + +TEST(ProcessorTest, AddHostsIntoZoneTest) { + fs::TempDir rootPath("/tmp/AddHostsIntoZoneTest.XXXXXX"); + auto kv = MockCluster::initMetaKV(rootPath.path()); + { + // Attempt to register heartbeat + const ClusterID kClusterId = 10; + for (auto i = 8987; i < 8990; i++) { + cpp2::HBReq req; + req.set_role(cpp2::HostRole::STORAGE); + req.set_host(HostAddr("127.0.0.1", i)); + req.set_cluster_id(kClusterId); + req.set_role(cpp2::HostRole::STORAGE); + auto* processor = HBProcessor::instance(kv.get(), nullptr, kClusterId); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND, resp.get_code()); + } + } + { + // Add host into zone with duplicate hosts + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_0"); + req.set_is_new(false); + std::vector hosts = {{"127.0.0.1", 8988}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } + { + // Add host into zone with empty hosts + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_0"); + req.set_is_new(false); + req.set_hosts({}); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } + { + // Add host into zone which zone is not exist. + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_not_existed"); + req.set_is_new(false); + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND, resp.get_code()); + } + { + // Add host into zone. + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_0"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Add host into zone with zone name conflict. + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_0"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); + } + { + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_1"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Add existed hosts. + cpp2::AddHostsReq req; + std::vector hosts; + hosts.emplace_back("127.0.0.1", 8988); + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); + } + { + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_1"); + req.set_is_new(false); + std::vector hosts = {{"127.0.0.1", 8976}, {"127.0.0.1", 8988}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); + } + { + // Drop hosts which is empty. + cpp2::DropHostsReq req; + std::vector hosts; + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } + { + // Drop hosts which have duplicate element. + cpp2::DropHostsReq req; + std::vector hosts = {{"127.0.0.1", 8988}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } + { + // Drop hosts. + cpp2::DropHostsReq req; + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Attempt to register heartbeat + const ClusterID kClusterId = 10; + for (auto i = 8987; i < 8990; i++) { + cpp2::HBReq req; + req.set_role(cpp2::HostRole::STORAGE); + req.set_host(HostAddr("127.0.0.1", i)); + req.set_cluster_id(kClusterId); + req.set_role(cpp2::HostRole::STORAGE); + auto* processor = HBProcessor::instance(kv.get(), nullptr, kClusterId); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND, resp.get_code()); + } + } +} + +TEST(ProcessorTest, DropHostsTest) { + fs::TempDir rootPath("/tmp/DropHostsTest.XXXXXX"); + auto kv = MockCluster::initMetaKV(rootPath.path()); + { + // Attempt to register heartbeat + const ClusterID kClusterId = 10; + for (auto i = 8987; i < 8990; i++) { + cpp2::HBReq req; + req.set_role(cpp2::HostRole::STORAGE); + req.set_host(HostAddr("127.0.0.1", i)); + req.set_cluster_id(kClusterId); + req.set_role(cpp2::HostRole::STORAGE); + auto* processor = HBProcessor::instance(kv.get(), nullptr, kClusterId); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND, resp.get_code()); + } + } + { + // Add multi host + cpp2::AddHostsReq req; + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Attempt to register heartbeat + const ClusterID kClusterId = 10; + for (auto i = 8987; i < 8990; i++) { + cpp2::HBReq req; + req.set_role(cpp2::HostRole::STORAGE); + req.set_host(HostAddr("127.0.0.1", i)); + req.set_cluster_id(kClusterId); + req.set_role(cpp2::HostRole::STORAGE); + auto* processor = HBProcessor::instance(kv.get(), nullptr, kClusterId); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + } + { + // Show the zones created by add hosts + cpp2::ListZonesReq req; + auto* processor = ListZonesProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + ASSERT_EQ(3, resp.get_zones().size()); + ASSERT_EQ("default_zone_127.0.0.1_8987", resp.get_zones()[0].get_zone_name()); + ASSERT_EQ("default_zone_127.0.0.1_8988", resp.get_zones()[1].get_zone_name()); + ASSERT_EQ("default_zone_127.0.0.1_8989", resp.get_zones()[2].get_zone_name()); + } + { + // Create Space on cluster, the replica number same with the zone size. + cpp2::SpaceDesc properties; + properties.set_space_name("default_space_0"); + properties.set_partition_num(9); + properties.set_replica_factor(3); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + cpp2::CreateSpaceReq req; + req.set_properties(std::move(properties)); + auto* processor = CreateSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + ASSERT_EQ(1, resp.get_id().get_space_id()); + } + { + // Create Space on cluster, the replica number less than the zone size. + cpp2::SpaceDesc properties; + properties.set_space_name("default_space_1"); + properties.set_partition_num(9); + properties.set_replica_factor(1); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + cpp2::CreateSpaceReq req; + req.set_properties(std::move(properties)); + auto* processor = CreateSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + ASSERT_EQ(2, resp.get_id().get_space_id()); + } + { + // Create Space on cluster, the replica number greater than the zone size. + cpp2::SpaceDesc properties; + properties.set_space_name("default_space_2"); + properties.set_partition_num(9); + properties.set_replica_factor(6); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + cpp2::CreateSpaceReq req; + req.set_properties(std::move(properties)); + auto* processor = CreateSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } + { + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_0"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8976}, {"127.0.0.1", 8977}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_1"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8978}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_2"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8979}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Show the zones created by add hosts + cpp2::ListZonesReq req; + auto* processor = ListZonesProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + ASSERT_EQ(6, resp.get_zones().size()); + ASSERT_EQ("default_zone_127.0.0.1_8987", resp.get_zones()[0].get_zone_name()); + ASSERT_EQ("default_zone_127.0.0.1_8988", resp.get_zones()[1].get_zone_name()); + ASSERT_EQ("default_zone_127.0.0.1_8989", resp.get_zones()[2].get_zone_name()); + ASSERT_EQ("zone_0", resp.get_zones()[3].get_zone_name()); + ASSERT_EQ("zone_1", resp.get_zones()[4].get_zone_name()); + ASSERT_EQ("zone_2", resp.get_zones()[5].get_zone_name()); + } + { + // Create Space on cluster, the replica number same with the zone size + cpp2::SpaceDesc properties; + properties.set_space_name("default_space_on_zone_3"); + properties.set_partition_num(9); + properties.set_replica_factor(3); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + std::vector zones = {"zone_0", "zone_1", "zone_2"}; + properties.set_zone_names(std::move(zones)); + cpp2::CreateSpaceReq req; + req.set_properties(std::move(properties)); + auto* processor = CreateSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + ASSERT_EQ(4, resp.get_id().get_space_id()); + } + { + // Create Space on cluster, the replica number less than the zone size + cpp2::SpaceDesc properties; + properties.set_space_name("default_space_on_zone_1"); + properties.set_partition_num(9); + properties.set_replica_factor(1); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + std::vector zones = {"zone_0"}; + properties.set_zone_names(std::move(zones)); + cpp2::CreateSpaceReq req; + req.set_properties(std::move(properties)); + auto* processor = CreateSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + ASSERT_EQ(5, resp.get_id().get_space_id()); + } + { + // Create Space on cluster, the replica number greater than the zone size + cpp2::SpaceDesc properties; + properties.set_space_name("default_space_on_zone_6"); + properties.set_partition_num(9); + properties.set_replica_factor(6); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + std::vector zones = {"zone_0"}; + properties.set_zone_names(std::move(zones)); + cpp2::CreateSpaceReq req; + req.set_properties(std::move(properties)); + auto* processor = CreateSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } + { + // Drop hosts which hold partition. + cpp2::DropHostsReq req; + std::vector hosts = {{"127.0.0.1", 8987}}; + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_CONFLICT, resp.get_code()); + } + { + // Drop hosts which hold partition. + cpp2::DropHostsReq req; + std::vector hosts = {{"127.0.0.1", 8977}}; + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_CONFLICT, resp.get_code()); + } + { + cpp2::DropSpaceReq req; + req.set_space_name("default_space_on_zone_1"); + auto* processor = DropSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + cpp2::DropSpaceReq req; + req.set_space_name("default_space_on_zone_3"); + auto* processor = DropSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + cpp2::DropSpaceReq req; + req.set_space_name("default_space_0"); + auto* processor = DropSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + cpp2::DropSpaceReq req; + req.set_space_name("default_space_1"); + auto* processor = DropSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Drop hosts. + cpp2::DropHostsReq req; + std::vector hosts = {{"127.0.0.1", 8987}}; + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Drop hosts. + cpp2::DropHostsReq req; + std::vector hosts = {{"127.0.0.1", 8976}}; + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Drop hosts. + cpp2::DropHostsReq req; + std::vector hosts = {{"127.0.0.1", 8978}}; + req.set_hosts(std::move(hosts)); + auto* processor = DropHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Show the zones created by add hosts + cpp2::ListZonesReq req; + auto* processor = ListZonesProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + ASSERT_EQ(4, resp.get_zones().size()); + ASSERT_EQ("default_zone_127.0.0.1_8988", resp.get_zones()[0].get_zone_name()); + ASSERT_EQ("default_zone_127.0.0.1_8989", resp.get_zones()[1].get_zone_name()); + ASSERT_EQ("zone_0", resp.get_zones()[2].get_zone_name()); + ASSERT_EQ("zone_2", resp.get_zones()[3].get_zone_name()); + } +} + +TEST(ProcessorTest, RenameZoneTest) { + fs::TempDir rootPath("/tmp/RenameZoneTest.XXXXXX"); + auto kv = MockCluster::initMetaKV(rootPath.path()); + { + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_0"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8987}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_1"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8988}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + cpp2::AddHostsIntoZoneReq req; + req.set_zone_name("zone_2"); + req.set_is_new(true); + std::vector hosts = {{"127.0.0.1", 8989}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsIntoZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + // Attempt to register heartbeat + const ClusterID kClusterId = 10; + for (auto i = 8987; i < 8990; i++) { + cpp2::HBReq req; + req.set_role(cpp2::HostRole::STORAGE); + req.set_host(HostAddr("127.0.0.1", i)); + req.set_cluster_id(kClusterId); + req.set_role(cpp2::HostRole::STORAGE); + auto* processor = HBProcessor::instance(kv.get(), nullptr, kClusterId); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + } + { + cpp2::ListZonesReq req; + auto* processor = ListZonesProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + auto zones = resp.get_zones(); + ASSERT_EQ(3, zones.size()); + ASSERT_EQ("zone_0", zones[0].get_zone_name()); + ASSERT_EQ("zone_1", zones[1].get_zone_name()); + ASSERT_EQ("zone_2", zones[2].get_zone_name()); + } + { + cpp2::SpaceDesc properties; + properties.set_space_name("default"); + properties.set_partition_num(9); + properties.set_replica_factor(3); + properties.set_charset_name("utf8"); + properties.set_collate_name("utf8_bin"); + std::vector zones = {"zone_0", "zone_1", "zone_2"}; + properties.set_zone_names(std::move(zones)); + cpp2::CreateSpaceReq req; + req.set_properties(std::move(properties)); + auto* processor = CreateSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + ASSERT_EQ(1, resp.get_id().get_space_id()); + } + { + cpp2::GetSpaceReq req; + req.set_space_name("default"); + auto* processor = GetSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + auto properties = resp.get_item().get_properties(); + ASSERT_EQ("default", properties.get_space_name()); + ASSERT_EQ(9, properties.get_partition_num()); + ASSERT_EQ(3, properties.get_replica_factor()); + ASSERT_EQ("utf8", properties.get_charset_name()); + ASSERT_EQ("utf8_bin", properties.get_collate_name()); + auto zones = properties.get_zone_names(); + ASSERT_EQ(3, zones.size()); + ASSERT_EQ("zone_0", zones[0]); + ASSERT_EQ("zone_1", zones[1]); + ASSERT_EQ("zone_2", zones[2]); + } + { + cpp2::RenameZoneReq req; + req.set_original_zone_name("zone_not_exist"); + req.set_zone_name("new_zone_name"); + auto* processor = RenameZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND, resp.get_code()); + } + { + cpp2::RenameZoneReq req; + req.set_original_zone_name("zone_0"); + req.set_zone_name("zone_1"); + auto* processor = RenameZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_EXISTED, resp.get_code()); + } + { + cpp2::RenameZoneReq req; + req.set_original_zone_name("zone_1"); + req.set_zone_name("z_1"); + auto* processor = RenameZoneProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + { + cpp2::GetSpaceReq req; + req.set_space_name("default"); + auto* processor = GetSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + auto properties = resp.get_item().get_properties(); + ASSERT_EQ("default", properties.get_space_name()); + ASSERT_EQ(9, properties.get_partition_num()); + ASSERT_EQ(3, properties.get_replica_factor()); + ASSERT_EQ("utf8", properties.get_charset_name()); + ASSERT_EQ("utf8_bin", properties.get_collate_name()); + auto zones = properties.get_zone_names(); + ASSERT_EQ(3, zones.size()); + ASSERT_EQ("zone_0", zones[0]); + ASSERT_EQ("z_1", zones[1]); + ASSERT_EQ("zone_2", zones[2]); + } +} + } // namespace meta } // namespace nebula diff --git a/src/meta/test/RestoreProcessorTest.cpp b/src/meta/test/RestoreProcessorTest.cpp index fa67ad2df63..704698a4605 100644 --- a/src/meta/test/RestoreProcessorTest.cpp +++ b/src/meta/test/RestoreProcessorTest.cpp @@ -108,7 +108,7 @@ TEST(RestoreProcessorTest, RestoreTest) { auto it = std::find_if(files.cbegin(), files.cend(), [](auto& f) { auto const pos = f.find_last_of("/"); auto name = f.substr(pos + 1); - if (name == MetaKeyUtils::groupPrefix() || name == MetaKeyUtils::zonePrefix()) { + if (name == MetaKeyUtils::zonePrefix()) { return true; } return false; @@ -188,11 +188,6 @@ TEST(RestoreProcessorTest, RestoreTest) { iter->next(); } - prefix = MetaKeyUtils::groupPrefix(); - result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); - // ASSERT_NE(nebula::cpp2::ErrorCode::SUCCEEDED, result); - ASSERT_FALSE(iter->valid()); - prefix = MetaKeyUtils::userPrefix(); result = kvRestore->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, result); @@ -315,7 +310,7 @@ TEST(RestoreProcessorTest, RestoreFullTest) { auto it = std::find_if(files.cbegin(), files.cend(), [](auto& f) { auto const pos = f.find_last_of("/"); auto name = f.substr(pos + 1); - if (name == MetaKeyUtils::groupPrefix() || name == MetaKeyUtils::zonePrefix()) { + if (name == MetaKeyUtils::zonePrefix()) { return true; } return false; diff --git a/src/meta/test/TestUtils.h b/src/meta/test/TestUtils.h index 46e26e67dd1..4d75c5382f2 100644 --- a/src/meta/test/TestUtils.h +++ b/src/meta/test/TestUtils.h @@ -38,7 +38,6 @@ using mock::MockCluster; using nebula::cpp2::PropertyType; using ZoneInfo = std::unordered_map>; -using GroupInfo = std::unordered_map>; ObjectPool metaTestPool; auto metaPool = &metaTestPool; @@ -84,6 +83,16 @@ class TestUtils { static int32_t createSomeHosts(kvstore::KVStore* kv, std::vector hosts = { {"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}) { + // Record machine information + std::vector machines; + for (auto& host : hosts) { + machines.emplace_back(nebula::MetaKeyUtils::machineKey(host.host, host.port), ""); + } + folly::Baton baton; + kv->asyncMultiPut( + kDefaultSpaceId, kDefaultPartId, std::move(machines), [&](auto) { baton.post(); }); + baton.wait(); + std::vector thriftHosts(hosts); registerHB(kv, hosts); { @@ -102,9 +111,7 @@ class TestUtils { return hosts.size(); } - static void assembleGroupAndZone(kvstore::KVStore* kv, - const ZoneInfo& zoneInfo, - const GroupInfo& groupInfo) { + static void assembleZone(kvstore::KVStore* kv, const ZoneInfo& zoneInfo) { std::vector data; int32_t id = 10; for (auto iter = zoneInfo.begin(); iter != zoneInfo.end(); iter++) { @@ -114,14 +121,6 @@ class TestUtils { id += 1; } - id = 100; - for (auto iter = groupInfo.begin(); iter != groupInfo.end(); iter++) { - data.emplace_back(MetaKeyUtils::indexGroupKey(iter->first), - std::string(reinterpret_cast(&id), sizeof(GroupID))); - data.emplace_back(MetaKeyUtils::groupKey(iter->first), MetaKeyUtils::groupVal(iter->second)); - id += 1; - } - folly::Baton baton; kv->asyncMultiPut(0, 0, std::move(data), [&](nebula::cpp2::ErrorCode code) { ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code); @@ -171,46 +170,6 @@ class TestUtils { return ret; } - static bool modifyZoneAboutGroup(kvstore::KVStore* kv, - const std::string& groupName, - const std::string& zoneName, - bool isAdd) { - auto groupKey = MetaKeyUtils::groupKey(groupName); - std::string groupValue; - auto retCode = kv->get(kDefaultSpaceId, kDefaultPartId, groupKey, &groupValue); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Get group " << groupName << " failed"; - return false; - } - - auto zoneNames = MetaKeyUtils::parseZoneNames(std::move(groupValue)); - auto iter = std::find(zoneNames.begin(), zoneNames.end(), zoneName); - if (isAdd) { - if (iter != zoneNames.end()) { - LOG(ERROR) << "Zone " << zoneName << " have existed"; - return false; - } - zoneNames.emplace_back(std::move(zoneName)); - } else { - if (iter == zoneNames.end()) { - LOG(ERROR) << "Zone " << zoneName << " not existed"; - return false; - } - zoneNames.erase(iter); - } - - std::vector data; - data.emplace_back(std::move(groupKey), MetaKeyUtils::groupVal(zoneNames)); - - folly::Baton baton; - kv->asyncMultiPut(0, 0, std::move(data), [&](nebula::cpp2::ErrorCode code) { - ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code); - baton.post(); - }); - baton.wait(); - return true; - } - static void assembleSpace(kvstore::KVStore* kv, GraphSpaceID id, int32_t partitionNum, diff --git a/src/parser/MaintainSentences.cpp b/src/parser/MaintainSentences.cpp index 1a90d068192..78cf358e57a 100644 --- a/src/parser/MaintainSentences.cpp +++ b/src/parser/MaintainSentences.cpp @@ -358,12 +358,29 @@ std::string ShowCreateEdgeIndexSentence::toString() const { return folly::stringPrintf("SHOW CREATE EDGE INDEX %s", indexName_.get()->c_str()); } -std::string AddZoneSentence::toString() const { +std::string AddHostsSentence::toString() const { + std::string buf; + buf.reserve(64); + buf += "ADD HOSTS "; + buf += hosts_->toString(); + return buf; +} + +std::string DropHostsSentence::toString() const { + std::string buf; + buf += "DROP HOSTS "; + buf += hosts_->toString(); + buf.reserve(64); + return buf; +} + +std::string MergeZoneSentence::toString() const { std::string buf; buf.reserve(128); - buf += "ADD ZONE "; + buf += "MERGE ZONE "; + buf += zoneNames_->toString(); + buf += " INTO "; buf += *zoneName_; - buf += hosts_->toString(); return buf; } @@ -371,28 +388,42 @@ std::string DropZoneSentence::toString() const { return folly::stringPrintf("DROP ZONE %s", zoneName_.get()->c_str()); } -std::string DescribeZoneSentence::toString() const { - return folly::stringPrintf("DESCRIBE ZONE %s", zoneName_.get()->c_str()); +std::string SplitZoneSentence::toString() const { + std::string buf; + buf.reserve(128); + buf += "SPLIT ZONE "; + buf += *zoneName_; + buf += " INTO "; + buf += zoneNames_->toString(); + return buf; } -std::string ListZonesSentence::toString() const { return folly::stringPrintf("SHOW ZONES"); } - -std::string AddHostIntoZoneSentence::toString() const { +std::string RenameZoneSentence::toString() const { std::string buf; - buf.reserve(64); - buf += "ADD HOST "; - buf += address_->toString(); - buf += " INTO ZONE "; + buf.reserve(128); + buf += "RENAME ZONE "; + buf += *originalZoneName_; + buf += " TO "; buf += *zoneName_; return buf; } -std::string DropHostFromZoneSentence::toString() const { +std::string DescribeZoneSentence::toString() const { + return folly::stringPrintf("DESCRIBE ZONE %s", zoneName_.get()->c_str()); +} + +std::string ListZonesSentence::toString() const { return folly::stringPrintf("SHOW ZONES"); } + +std::string AddHostsIntoZoneSentence::toString() const { std::string buf; buf.reserve(64); - buf += "DROP HOST "; + buf += "ADD HOSTS "; buf += address_->toString(); - buf += " FROM ZONE "; + if (isNew_) { + buf += " INTO NEW ZONE "; + } else { + buf += " INTO ZONE "; + } buf += *zoneName_; return buf; } diff --git a/src/parser/MaintainSentences.h b/src/parser/MaintainSentences.h index bc5de67c9a2..ee7d6dde864 100644 --- a/src/parser/MaintainSentences.h +++ b/src/parser/MaintainSentences.h @@ -754,45 +754,60 @@ class ShowCreateEdgeIndexSentence : public Sentence { std::unique_ptr indexName_; }; -class AddZoneSentence : public Sentence { +class AddHostsSentence : public Sentence { public: - explicit AddZoneSentence(std::string *zoneName, HostList *hosts) { - zoneName_.reset(zoneName); + explicit AddHostsSentence(HostList *hosts) { hosts_.reset(hosts); - kind_ = Kind::kAddZone; + kind_ = Kind::kAddHosts; } + const HostList *hosts() const { return hosts_.get(); } + std::string toString() const override; - const std::string *zoneName() const { return zoneName_.get(); } + private: + std::unique_ptr hosts_; +}; + +class DropHostsSentence : public Sentence { + public: + explicit DropHostsSentence(HostList *hosts) { + hosts_.reset(hosts); + kind_ = Kind::kDropHosts; + } const HostList *hosts() const { return hosts_.get(); } + std::string toString() const override; + private: - std::unique_ptr zoneName_; std::unique_ptr hosts_; }; -class DropZoneSentence : public Sentence { +class MergeZoneSentence : public Sentence { public: - explicit DropZoneSentence(std::string *zoneName) { + explicit MergeZoneSentence(ZoneNameList *zoneNames, std::string *zoneName) { zoneName_.reset(zoneName); - kind_ = Kind::kDropZone; + zoneNames_.reset(zoneNames); + kind_ = Kind::kMergeZone; } std::string toString() const override; const std::string *zoneName() const { return zoneName_.get(); } + const ZoneNameList *zoneNames() const { return zoneNames_.get(); } + private: std::unique_ptr zoneName_; + std::unique_ptr zoneNames_; }; -class DescribeZoneSentence : public Sentence { +class DropZoneSentence : public Sentence { public: - explicit DescribeZoneSentence(std::string *zoneName) { + explicit DropZoneSentence(std::string *zoneName) { zoneName_.reset(zoneName); - kind_ = Kind::kDescribeZone; + kind_ = Kind::kDropZone; } std::string toString() const override; @@ -803,49 +818,87 @@ class DescribeZoneSentence : public Sentence { std::unique_ptr zoneName_; }; -class ListZonesSentence : public Sentence { +class SplitZoneSentence : public Sentence { public: - ListZonesSentence() { kind_ = Kind::kListZones; } + explicit SplitZoneSentence(std::string *zoneName, ZoneNameList *zoneNames) { + zoneName_.reset(zoneName); + zoneNames_.reset(zoneNames); + kind_ = Kind::kSplitZone; + } std::string toString() const override; + + const std::string *zoneName() const { return zoneName_.get(); } + + const ZoneNameList *zoneNames() const { return zoneNames_.get(); } + + private: + std::unique_ptr zoneName_; + std::unique_ptr zoneNames_; }; -class AddHostIntoZoneSentence : public Sentence { +class RenameZoneSentence : public Sentence { public: - explicit AddHostIntoZoneSentence(HostAddr *address, std::string *zoneName) { - address_.reset(address); + explicit RenameZoneSentence(std::string *originalZoneName, std::string *zoneName) { + originalZoneName_.reset(originalZoneName); zoneName_.reset(zoneName); - kind_ = Kind::kAddHostIntoZone; + kind_ = Kind::kRenameZone; } + std::string toString() const override; + + const std::string *originalZoneName() const { return originalZoneName_.get(); } + const std::string *zoneName() const { return zoneName_.get(); } - const HostAddr *address() const { return address_.get(); } + private: + std::unique_ptr originalZoneName_; + std::unique_ptr zoneName_; +}; + +class DescribeZoneSentence : public Sentence { + public: + explicit DescribeZoneSentence(std::string *zoneName) { + zoneName_.reset(zoneName); + kind_ = Kind::kDescribeZone; + } std::string toString() const override; + const std::string *zoneName() const { return zoneName_.get(); } + private: std::unique_ptr zoneName_; - std::unique_ptr address_; }; -class DropHostFromZoneSentence : public Sentence { +class ListZonesSentence : public Sentence { + public: + ListZonesSentence() { kind_ = Kind::kListZones; } + + std::string toString() const override; +}; + +class AddHostsIntoZoneSentence : public Sentence { public: - explicit DropHostFromZoneSentence(HostAddr *address, std::string *zoneName) { + AddHostsIntoZoneSentence(HostList *address, std::string *zoneName, bool isNew) { address_.reset(address); zoneName_.reset(zoneName); - kind_ = Kind::kDropHostFromZone; + isNew_ = isNew; + kind_ = Kind::kAddHostsIntoZone; } const std::string *zoneName() const { return zoneName_.get(); } - const HostAddr *address() const { return address_.get(); } + const HostList *address() const { return address_.get(); } + + bool isNew() const { return isNew_; } std::string toString() const override; private: std::unique_ptr zoneName_; - std::unique_ptr address_; + std::unique_ptr address_; + bool isNew_; }; #ifndef FULLTEXT_INDEX_NAME_PREFIX diff --git a/src/parser/Sentence.h b/src/parser/Sentence.h index 2464d581715..9af5ebe410d 100644 --- a/src/parser/Sentence.h +++ b/src/parser/Sentence.h @@ -55,6 +55,8 @@ class Sentence { kUpdateVertex, kInsertEdges, kUpdateEdge, + kAddHosts, + kDropHosts, kShowHosts, kShowSpaces, kShowParts, @@ -112,12 +114,13 @@ class Sentence { kAdminJob, kAdminShowJobs, kGetSubgraph, - kAddZone, + kMergeZone, + kRenameZone, kDropZone, + kSplitZone, kDescribeZone, kListZones, - kAddHostIntoZone, - kDropHostFromZone, + kAddHostsIntoZone, kAddListener, kRemoveListener, kShowListener, diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 9febad281ea..061d8993f2b 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -193,7 +193,7 @@ static constexpr size_t kCommentLengthLimit = 256; %token KW_STARTS KW_ENDS %token KW_UNWIND KW_SKIP KW_OPTIONAL %token KW_CASE KW_THEN KW_ELSE KW_END -%token KW_GROUP KW_ZONE KW_GROUPS KW_ZONES KW_INTO +%token KW_GROUP KW_ZONE KW_GROUPS KW_ZONES KW_INTO KW_NEW %token KW_LISTENER KW_ELASTICSEARCH KW_FULLTEXT KW_HTTPS KW_HTTP %token KW_AUTO KW_FUZZY KW_PREFIX KW_REGEXP KW_WILDCARD %token KW_TEXT KW_SEARCH KW_CLIENTS KW_SIGN KW_SERVICE KW_TEXT_SEARCH @@ -203,6 +203,7 @@ static constexpr size_t kCommentLengthLimit = 256; %token KW_KILL KW_QUERY KW_QUERIES KW_TOP %token KW_GEOGRAPHY KW_POINT KW_LINESTRING KW_POLYGON %token KW_LIST KW_MAP +%token KW_MERGE KW_SPLIT KW_RENAME /* symbols */ %token L_PAREN R_PAREN L_BRACKET R_BRACKET L_BRACE R_BRACE COMMA @@ -355,8 +356,9 @@ static constexpr size_t kCommentLengthLimit = 256; %type drop_tag_index_sentence drop_edge_index_sentence drop_fulltext_index_sentence %type describe_tag_index_sentence describe_edge_index_sentence %type rebuild_tag_index_sentence rebuild_edge_index_sentence rebuild_fulltext_index_sentence -%type add_zone_sentence drop_zone_sentence desc_zone_sentence -%type add_host_into_zone_sentence drop_host_from_zone_sentence +%type add_hosts_sentence drop_hosts_sentence +%type drop_zone_sentence desc_zone_sentence +%type merge_zone_sentence split_zone_sentence rename_zone_sentence %type create_snapshot_sentence drop_snapshot_sentence %type add_listener_sentence remove_listener_sentence list_listener_sentence @@ -500,6 +502,7 @@ unreserved_keyword | KW_ELSE { $$ = new std::string("else"); } | KW_END { $$ = new std::string("end"); } | KW_INTO { $$ = new std::string("into"); } + | KW_NEW { $$ = new std::string("new"); } | KW_GROUPS { $$ = new std::string("groups"); } | KW_ZONE { $$ = new std::string("zone"); } | KW_ZONES { $$ = new std::string("zones"); } @@ -534,6 +537,9 @@ unreserved_keyword | KW_POLYGON { $$ = new std::string("polygon"); } | KW_HTTP { $$ = new std::string("http"); } | KW_HTTPS { $$ = new std::string("https"); } + | KW_MERGE { $$ = new std::string("merge"); } + | KW_SPLIT { $$ = new std::string("split"); } + | KW_RENAME { $$ = new std::string("rename"); } ; expression @@ -2674,11 +2680,30 @@ rebuild_fulltext_index_sentence $$ = new AdminJobSentence(meta::cpp2::AdminJobOp::ADD, meta::cpp2::AdminCmd::REBUILD_FULLTEXT_INDEX); } + ; +add_hosts_sentence + : KW_ADD KW_HOSTS host_list { + $$ = new AddHostsSentence($3); + } + | KW_ADD KW_HOSTS host_list KW_INTO KW_ZONE name_label { + $$ = new AddHostsIntoZoneSentence($3, $6, false); + } + | KW_ADD KW_HOSTS host_list KW_INTO KW_NEW KW_ZONE name_label { + $$ = new AddHostsIntoZoneSentence($3, $7, true); + } + ; -add_zone_sentence - : KW_ADD KW_ZONE name_label host_list { - $$ = new AddZoneSentence($3, $4); +drop_hosts_sentence + : KW_DROP KW_HOSTS host_list { + $$ = new DropHostsSentence($3); + } + ; + + +merge_zone_sentence + : KW_MERGE KW_ZONE zone_name_list KW_INTO name_label { + $$ = new MergeZoneSentence($3, $5); } ; @@ -2688,24 +2713,24 @@ drop_zone_sentence } ; -desc_zone_sentence - : KW_DESCRIBE KW_ZONE name_label { - $$ = new DescribeZoneSentence($3); - } - | KW_DESC KW_ZONE name_label { - $$ = new DescribeZoneSentence($3); +split_zone_sentence + : KW_SPLIT KW_ZONE name_label KW_FROM zone_name_list { + $$ = new SplitZoneSentence($3, $5); } ; -add_host_into_zone_sentence - : KW_ADD KW_HOST host_item KW_INTO KW_ZONE name_label { - $$ = new AddHostIntoZoneSentence($3, $6); +rename_zone_sentence + : KW_RENAME KW_ZONE name_label KW_TO name_label { + $$ = new RenameZoneSentence($3, $5); } ; -drop_host_from_zone_sentence - : KW_DROP KW_HOST host_item KW_FROM KW_ZONE name_label { - $$ = new DropHostFromZoneSentence($3, $6); +desc_zone_sentence + : KW_DESCRIBE KW_ZONE name_label { + $$ = new DescribeZoneSentence($3); + } + | KW_DESC KW_ZONE name_label { + $$ = new DescribeZoneSentence($3); } ; @@ -3650,11 +3675,13 @@ maintain_sentence | rebuild_tag_index_sentence { $$ = $1; } | rebuild_edge_index_sentence { $$ = $1; } | rebuild_fulltext_index_sentence { $$ = $1; } - | add_zone_sentence { $$ = $1; } + | add_hosts_sentence { $$ = $1; } + | drop_hosts_sentence { $$ = $1; } + | merge_zone_sentence { $$ = $1; } | drop_zone_sentence { $$ = $1; } + | split_zone_sentence { $$ = $1; } + | rename_zone_sentence { $$ = $1; } | desc_zone_sentence { $$ = $1; } - | add_host_into_zone_sentence { $$ = $1; } - | drop_host_from_zone_sentence { $$ = $1; } | show_sentence { $$ = $1; } | create_user_sentence { $$ = $1; } | alter_user_sentence { $$ = $1; } diff --git a/src/parser/scanner.lex b/src/parser/scanner.lex index 4fe773332d2..1a4e4a0a033 100644 --- a/src/parser/scanner.lex +++ b/src/parser/scanner.lex @@ -237,6 +237,7 @@ CHINESE_LABEL ({U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U})+ "ZONE" { return TokenType::KW_ZONE; } "ZONES" { return TokenType::KW_ZONES; } "INTO" { return TokenType::KW_INTO; } +"NEW" { return TokenType::KW_NEW; } "LISTENER" { return TokenType::KW_LISTENER; } "ELASTICSEARCH" { return TokenType::KW_ELASTICSEARCH; } "HTTP" { return TokenType::KW_HTTP; } @@ -267,6 +268,10 @@ CHINESE_LABEL ({U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U})+ "POINT" { return TokenType::KW_POINT; } "LINESTRING" { return TokenType::KW_LINESTRING; } "POLYGON" { return TokenType::KW_POLYGON; } +"MERGE" { return TokenType::KW_MERGE; } +"SPLIT" { return TokenType::KW_SPLIT; } +"RENAME" { return TokenType::KW_RENAME; } + "TRUE" { yylval->boolval = true; return TokenType::BOOL; } "FALSE" { yylval->boolval = false; return TokenType::BOOL; } diff --git a/src/parser/test/ParserTest.cpp b/src/parser/test/ParserTest.cpp index 940686fe6b7..1fa8d794979 100644 --- a/src/parser/test/ParserTest.cpp +++ b/src/parser/test/ParserTest.cpp @@ -1425,6 +1425,30 @@ TEST_F(ParserTest, subgraph) { } TEST_F(ParserTest, AdminOperation) { + { + GQLParser parser; + std::string query = "ADD HOSTS 127.0.0.1:1000"; + auto result = parser.parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + GQLParser parser; + std::string query = "ADD HOSTS 127.0.0.1:1000,127.0.0.1:9000"; + auto result = parser.parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + GQLParser parser; + std::string query = "DROP HOSTS 127.0.0.1:1000"; + auto result = parser.parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + GQLParser parser; + std::string query = "DROP HOSTS 127.0.0.1:1000,127.0.0.1:9000"; + auto result = parser.parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } { std::string query = "SHOW HOSTS"; auto result = parse(query); @@ -2709,6 +2733,29 @@ TEST_F(ParserTest, MatchListSubscriptRange) { } } +TEST_F(ParserTest, HOST) { + { + std::string query = "ADD HOSTS 127.0.0.1:8989"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "ADD HOSTS 127.0.0.1:8988,127.0.0.1:8989"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "DROP HOSTS 127.0.0.1:8989"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "DROP HOSTS 127.0.0.1:8988,127.0.0.1:8989"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } +} + TEST_F(ParserTest, Zone) { { std::string query = "SHOW ZONES"; @@ -2716,17 +2763,17 @@ TEST_F(ParserTest, Zone) { ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "ADD ZONE zone_0 127.0.0.1:8989,127.0.0.1:8988,127.0.0.1:8987"; + std::string query = "SHOW ZONES"; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "ADD HOST 127.0.0.1:8989 INTO ZONE zone_0"; + std::string query = "ADD HOSTS 127.0.0.1:8989 INTO ZONE zone_0"; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "DROP HOST 127.0.0.1:8989 FROM ZONE zone_0"; + std::string query = "ADD HOSTS 127.0.0.1:8988,127.0.0.1:8989 INTO ZONE zone_0"; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } @@ -2745,6 +2792,21 @@ TEST_F(ParserTest, Zone) { auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } + { + std::string query = "MERGE ZONE zone_1,zone_2 INTO zone"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "MERGE ZONE zone_1,zone_2 INTO zone_1"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "RENAME ZONE old_name TO new_name"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } } TEST_F(ParserTest, FullText) { diff --git a/src/parser/test/ScannerTest.cpp b/src/parser/test/ScannerTest.cpp index f842233d960..66b7a3cf16a 100644 --- a/src/parser/test/ScannerTest.cpp +++ b/src/parser/test/ScannerTest.cpp @@ -461,6 +461,9 @@ TEST(Scanner, Basic) { CHECK_SEMANTIC_TYPE("INTO", TokenType::KW_INTO), CHECK_SEMANTIC_TYPE("Into", TokenType::KW_INTO), CHECK_SEMANTIC_TYPE("into", TokenType::KW_INTO), + CHECK_SEMANTIC_TYPE("NEW", TokenType::KW_NEW), + CHECK_SEMANTIC_TYPE("New", TokenType::KW_NEW), + CHECK_SEMANTIC_TYPE("new", TokenType::KW_NEW), CHECK_SEMANTIC_TYPE("STATS", TokenType::KW_STATS), CHECK_SEMANTIC_TYPE("Stats", TokenType::KW_STATS), CHECK_SEMANTIC_TYPE("stats", TokenType::KW_STATS), @@ -500,6 +503,15 @@ TEST(Scanner, Basic) { CHECK_SEMANTIC_TYPE("TOP", TokenType::KW_TOP), CHECK_SEMANTIC_TYPE("Top", TokenType::KW_TOP), CHECK_SEMANTIC_TYPE("top", TokenType::KW_TOP), + CHECK_SEMANTIC_TYPE("MERGE", TokenType::KW_MERGE), + CHECK_SEMANTIC_TYPE("Merge", TokenType::KW_MERGE), + CHECK_SEMANTIC_TYPE("Merge", TokenType::KW_MERGE), + CHECK_SEMANTIC_TYPE("SPLIT", TokenType::KW_SPLIT), + CHECK_SEMANTIC_TYPE("Split", TokenType::KW_SPLIT), + CHECK_SEMANTIC_TYPE("split", TokenType::KW_SPLIT), + CHECK_SEMANTIC_TYPE("RENAME", TokenType::KW_RENAME), + CHECK_SEMANTIC_TYPE("Rename", TokenType::KW_RENAME), + CHECK_SEMANTIC_TYPE("rename", TokenType::KW_RENAME), CHECK_SEMANTIC_TYPE("_type", TokenType::TYPE_PROP), CHECK_SEMANTIC_TYPE("_id", TokenType::ID_PROP), diff --git a/src/storage/test/KVClientTest.cpp b/src/storage/test/KVClientTest.cpp index 06122bf9ca1..9be13c4a762 100644 --- a/src/storage/test/KVClientTest.cpp +++ b/src/storage/test/KVClientTest.cpp @@ -10,6 +10,7 @@ #include "common/datatypes/KeyValue.h" #include "common/fs/TempDir.h" #include "common/network/NetworkUtils.h" +#include "common/utils/MetaKeyUtils.h" #include "meta/test/TestUtils.h" #include "storage/test/TestUtils.h" @@ -40,6 +41,14 @@ TEST(KVClientTest, SimpleTest) { HostAddr storageAddr{storageName, storagePort}; cluster.startMeta(metaPath.path()); + folly::Baton baton; + auto* kv = cluster.metaKV_.get(); + std::vector machines; + machines.emplace_back(nebula::MetaKeyUtils::machineKey(storageName, storagePort), ""); + kv->asyncMultiPut( + kDefaultSpaceId, kDefaultPartId, std::move(machines), [&](auto) { baton.post(); }); + baton.wait(); + meta::MetaClientOptions options; options.localHost_ = storageAddr; options.role_ = meta::cpp2::HostRole::STORAGE; diff --git a/src/tools/meta-dump/MetaDumpTool.cpp b/src/tools/meta-dump/MetaDumpTool.cpp index e0f49db5d2f..23e4ff55c60 100644 --- a/src/tools/meta-dump/MetaDumpTool.cpp +++ b/src/tools/meta-dump/MetaDumpTool.cpp @@ -151,23 +151,6 @@ class MetaDumper { iter->Next(); } } - { - LOG(INFO) << "Group info"; - prefix = "__groups__"; - iter->Seek(rocksdb::Slice(prefix)); - while (iter->Valid() && iter->key().starts_with(prefix)) { - auto key = folly::StringPiece(iter->key().data(), iter->key().size()); - auto val = folly::StringPiece(iter->value().data(), iter->value().size()); - auto group = MetaKeyUtils::parseGroupName(key); - auto zones = MetaKeyUtils::parseZoneNames(val); - std::stringstream ss; - for (const auto& zone : zones) { - ss << zone << " "; - } - LOG(INFO) << folly::sformat("group name: {}, contain zones: {}", group, ss.str()); - iter->Next(); - } - } { LOG(INFO) << "Zone info"; prefix = "__zones__"; From 8588bbc94a1e5c24b028b06eed2ccf41f3a67ec4 Mon Sep 17 00:00:00 2001 From: "darion.yaphet" Date: Sat, 27 Nov 2021 06:41:18 +0800 Subject: [PATCH 3/5] fix test case --- src/graph/executor/admin/AddHostsExecutor.cpp | 3 +- src/graph/executor/admin/AddHostsExecutor.h | 3 +- .../executor/admin/DropHostsExecutor.cpp | 3 +- src/graph/executor/admin/DropHostsExecutor.h | 3 +- src/graph/executor/admin/ZoneExecutor.cpp | 13 +- src/meta/CMakeLists.txt | 2 + src/meta/MetaServiceHandler.cpp | 12 + src/meta/MetaServiceHandler.h | 5 + .../parts/CreateSpaceAsProcessor.cpp | 13 +- .../processors/parts/CreateSpaceProcessor.cpp | 238 ++++++++---------- .../processors/parts/CreateSpaceProcessor.h | 2 - .../zone/AddHostsIntoZoneProcessor.cpp | 3 +- .../processors/zone/AddHostsProcessor.cpp | 5 +- src/meta/processors/zone/AddHostsProcessor.h | 3 +- .../processors/zone/DropHostsProcessor.cpp | 3 +- src/meta/processors/zone/DropHostsProcessor.h | 3 +- .../processors/zone/MergeZoneProcessor.cpp | 14 ++ src/meta/processors/zone/MergeZoneProcessor.h | 29 +++ .../processors/zone/SplitZoneProcessor.cpp | 14 ++ src/meta/processors/zone/SplitZoneProcessor.h | 29 +++ src/meta/test/AuthProcessorTest.cpp | 12 +- src/meta/test/IndexProcessorTest.cpp | 13 +- src/meta/test/MetaClientTest.cpp | 218 +++++++++------- src/meta/test/ProcessorTest.cpp | 73 +++++- src/meta/test/TestUtils.h | 1 + src/parser/parser.yy | 14 +- src/parser/scanner.lex | 1 - src/parser/test/ScannerTest.cpp | 3 - src/storage/test/KVClientTest.cpp | 18 +- tests/admin/test_space.py | 21 +- tests/common/nebula_service.py | 25 ++ tests/maintain/test_zone.py | 64 +++-- tests/tck/features/schema/Comment.feature | 24 +- 33 files changed, 553 insertions(+), 334 deletions(-) create mode 100644 src/meta/processors/zone/MergeZoneProcessor.cpp create mode 100644 src/meta/processors/zone/MergeZoneProcessor.h create mode 100644 src/meta/processors/zone/SplitZoneProcessor.cpp create mode 100644 src/meta/processors/zone/SplitZoneProcessor.h diff --git a/src/graph/executor/admin/AddHostsExecutor.cpp b/src/graph/executor/admin/AddHostsExecutor.cpp index 9ca6da0232e..d24706b114f 100644 --- a/src/graph/executor/admin/AddHostsExecutor.cpp +++ b/src/graph/executor/admin/AddHostsExecutor.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. + * This source code is licensed under Apache 2.0 License. */ #include "graph/executor/admin/AddHostsExecutor.h" diff --git a/src/graph/executor/admin/AddHostsExecutor.h b/src/graph/executor/admin/AddHostsExecutor.h index 39506591b6a..833f430e816 100644 --- a/src/graph/executor/admin/AddHostsExecutor.h +++ b/src/graph/executor/admin/AddHostsExecutor.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. + * This source code is licensed under Apache 2.0 License. */ #ifndef GRAPH_EXECUTOR_ADMIN_ADD_HOST_EXECUTOR_H_ diff --git a/src/graph/executor/admin/DropHostsExecutor.cpp b/src/graph/executor/admin/DropHostsExecutor.cpp index 86f775eecf5..db51555efd5 100644 --- a/src/graph/executor/admin/DropHostsExecutor.cpp +++ b/src/graph/executor/admin/DropHostsExecutor.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. + * This source code is licensed under Apache 2.0 License. */ #include "graph/executor/admin/DropHostsExecutor.h" diff --git a/src/graph/executor/admin/DropHostsExecutor.h b/src/graph/executor/admin/DropHostsExecutor.h index d8285065063..f2fc896c0fb 100644 --- a/src/graph/executor/admin/DropHostsExecutor.h +++ b/src/graph/executor/admin/DropHostsExecutor.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. + * This source code is licensed under Apache 2.0 License. */ #ifndef GRAPH_EXECUTOR_ADMIN_DROP_HOST_EXECUTOR_H_ diff --git a/src/graph/executor/admin/ZoneExecutor.cpp b/src/graph/executor/admin/ZoneExecutor.cpp index 3a5e9938039..270c4847bee 100644 --- a/src/graph/executor/admin/ZoneExecutor.cpp +++ b/src/graph/executor/admin/ZoneExecutor.cpp @@ -18,7 +18,18 @@ folly::Future MergeZoneExecutor::execute() { folly::Future RenameZoneExecutor::execute() { SCOPED_TIMER(&execTime_); - return Status::OK(); + auto *rzNode = asNode(node()); + return qctx() + ->getMetaClient() + ->renameZone(rzNode->originalZoneName(), rzNode->zoneName()) + .via(runner()) + .thenValue([](StatusOr resp) { + if (!resp.ok()) { + LOG(ERROR) << "Rename Zone Failed :" << resp.status(); + return resp.status(); + } + return Status::OK(); + }); } folly::Future DropZoneExecutor::execute() { diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index 59737758909..89bb7a28411 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -86,6 +86,8 @@ nebula_add_library( processors/zone/DropZoneProcessor.cpp processors/zone/RenameZoneProcessor.cpp processors/zone/GetZoneProcessor.cpp + processors/zone/MergeZoneProcessor.cpp + processors/zone/SplitZoneProcessor.cpp processors/zone/ListZonesProcessor.cpp processors/listener/ListenerProcessor.cpp processors/session/SessionManagerProcessor.cpp diff --git a/src/meta/MetaServiceHandler.cpp b/src/meta/MetaServiceHandler.cpp index 4eda958934a..cc78fcbf2c6 100644 --- a/src/meta/MetaServiceHandler.cpp +++ b/src/meta/MetaServiceHandler.cpp @@ -67,7 +67,9 @@ #include "meta/processors/zone/DropZoneProcessor.h" #include "meta/processors/zone/GetZoneProcessor.h" #include "meta/processors/zone/ListZonesProcessor.h" +#include "meta/processors/zone/MergeZoneProcessor.h" #include "meta/processors/zone/RenameZoneProcessor.h" +#include "meta/processors/zone/SplitZoneProcessor.h" #define RETURN_FUTURE(processor) \ auto f = processor->getFuture(); \ @@ -445,6 +447,16 @@ folly::Future MetaServiceHandler::future_getZone(const cpp2:: RETURN_FUTURE(processor); } +folly::Future MetaServiceHandler::future_mergeZone(const cpp2::MergeZoneReq& req) { + auto* processor = MergeZoneProcessor::instance(kvstore_); + RETURN_FUTURE(processor); +} + +folly::Future MetaServiceHandler::future_splitZone(const cpp2::SplitZoneReq& req) { + auto* processor = SplitZoneProcessor::instance(kvstore_); + RETURN_FUTURE(processor); +} + folly::Future MetaServiceHandler::future_listZones( const cpp2::ListZonesReq& req) { auto* processor = ListZonesProcessor::instance(kvstore_); diff --git a/src/meta/MetaServiceHandler.h b/src/meta/MetaServiceHandler.h index 87bb22bfa7b..a1d7332fbf8 100644 --- a/src/meta/MetaServiceHandler.h +++ b/src/meta/MetaServiceHandler.h @@ -186,6 +186,10 @@ class MetaServiceHandler final : public cpp2::MetaServiceSvIf { folly::Future future_getZone(const cpp2::GetZoneReq& req) override; + folly::Future future_mergeZone(const cpp2::MergeZoneReq& req) override; + + folly::Future future_splitZone(const cpp2::SplitZoneReq& req) override; + folly::Future future_listZones(const cpp2::ListZonesReq& req) override; folly::Future future_addHostsIntoZone( @@ -210,6 +214,7 @@ class MetaServiceHandler final : public cpp2::MetaServiceSvIf { folly::Future future_getMetaDirInfo( const cpp2::GetMetaDirInfoReq& req) override; + folly::Future future_createSession( const cpp2::CreateSessionReq& req) override; diff --git a/src/meta/processors/parts/CreateSpaceAsProcessor.cpp b/src/meta/processors/parts/CreateSpaceAsProcessor.cpp index f6b27138b82..2533539e75c 100644 --- a/src/meta/processors/parts/CreateSpaceAsProcessor.cpp +++ b/src/meta/processors/parts/CreateSpaceAsProcessor.cpp @@ -47,7 +47,6 @@ void CreateSpaceAsProcessor::process(const cpp2::CreateSpaceAsReq &req) { } std::vector data; - auto newSpaceData = makeNewSpaceData(nebula::value(oldSpaceId), nebula::value(newSpaceId), newSpaceName); if (nebula::ok(newSpaceData)) { @@ -123,9 +122,10 @@ ErrorOr> CreateSpaceAsProcesso return nebula::error(partPrefix); } auto iter = nebula::value(partPrefix).get(); - for (; iter->valid(); iter->next()) { + while (iter->valid()) { auto partId = MetaKeyUtils::parsePartKeyPartId(iter->key()); data.emplace_back(MetaKeyUtils::partKey(newSpaceId, partId), iter->val()); + iter->next(); } return data; } @@ -145,7 +145,7 @@ ErrorOr> CreateSpaceAsProcesso std::vector data; auto iter = nebula::value(tagPrefix).get(); - for (; iter->valid(); iter->next()) { + while (iter->valid()) { auto val = iter->val(); auto tagId = MetaKeyUtils::parseTagId(iter->key()); @@ -157,6 +157,7 @@ ErrorOr> CreateSpaceAsProcesso auto tagVer = MetaKeyUtils::parseTagVersion(iter->key()); auto key = MetaKeyUtils::schemaTagKey(newSpaceId, tagId, tagVer); data.emplace_back(std::move(key), val.str()); + iter->next(); } return data; } @@ -176,7 +177,7 @@ ErrorOr> CreateSpaceAsProcesso std::vector data; auto iter = nebula::value(edgePrefix).get(); - for (; iter->valid(); iter->next()) { + while (iter->valid()) { auto val = iter->val(); auto edgeType = MetaKeyUtils::parseEdgeType(iter->key()); @@ -188,6 +189,7 @@ ErrorOr> CreateSpaceAsProcesso auto ver = MetaKeyUtils::parseEdgeVersion(iter->key()); auto key = MetaKeyUtils::schemaEdgeKey(newSpaceId, edgeType, ver); data.emplace_back(std::move(key), val.str()); + iter->next(); } return data; } @@ -207,7 +209,7 @@ ErrorOr> CreateSpaceAsProcesso std::vector data; auto iter = nebula::value(indexPrefix).get(); - for (; iter->valid(); iter->next()) { + while (iter->valid()) { auto val = iter->val(); auto indexId = MetaKeyUtils::parseIndexesKeyIndexID(iter->key()); @@ -219,6 +221,7 @@ ErrorOr> CreateSpaceAsProcesso std::string(reinterpret_cast(&indexId), sizeof(indexId))); data.emplace_back(MetaKeyUtils::indexKey(newSpaceId, indexId), MetaKeyUtils::indexVal(idxItem)); + iter->next(); } return data; } diff --git a/src/meta/processors/parts/CreateSpaceProcessor.cpp b/src/meta/processors/parts/CreateSpaceProcessor.cpp index 9a4bd792c33..ecaf159b58a 100644 --- a/src/meta/processors/parts/CreateSpaceProcessor.cpp +++ b/src/meta/processors/parts/CreateSpaceProcessor.cpp @@ -61,6 +61,7 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { // storage properties.set_partition_num(partitionNum); } + if (replicaFactor == 0) { replicaFactor = FLAGS_default_replica_factor; if (replicaFactor <= 0) { @@ -73,12 +74,14 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { // storage properties.set_replica_factor(replicaFactor); } + if (vidSize == 0) { LOG(ERROR) << "Create Space Failed : vid_size is illegal: " << vidSize; handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); onFinished(); return; } + if (vidType != nebula::cpp2::PropertyType::INT64 && vidType != nebula::cpp2::PropertyType::FIXED_STRING) { LOG(ERROR) << "Create Space Failed : vid_type is illegal: " @@ -87,6 +90,7 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { onFinished(); return; } + if (vidType == nebula::cpp2::PropertyType::INT64 && vidSize != 8) { LOG(ERROR) << "Create Space Failed : vid_size should be 8 if vid type is integer: " << vidSize; handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); @@ -105,148 +109,139 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { auto spaceId = nebula::value(idRet); std::vector data; - data.emplace_back(MetaKeyUtils::indexSpaceKey(spaceName), - std::string(reinterpret_cast(&spaceId), sizeof(spaceId))); - data.emplace_back(MetaKeyUtils::spaceKey(spaceId), MetaKeyUtils::spaceVal(properties)); - + std::vector<::std::string> zones; nebula::cpp2::ErrorCode code = nebula::cpp2::ErrorCode::SUCCEEDED; - if (!properties.get_zone_names().empty()) { - auto zones = properties.get_zone_names(); - for (auto& zone : zones) { - auto zoneKey = MetaKeyUtils::zoneKey(zone); - auto ret = doGet(zoneKey); - if (!nebula::ok(ret)) { - auto retCode = nebula::error(ret); - if (retCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { - code = nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND; - } - LOG(ERROR) << " Get Zone Name: " << zone << " failed."; - break; - } - } - - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Create space failed"; + if (properties.get_zone_names().empty()) { + const auto& zonePrefix = MetaKeyUtils::zonePrefix(); + auto zoneIterRet = doPrefix(zonePrefix); + if (!nebula::ok(zoneIterRet)) { + code = nebula::error(zoneIterRet); + LOG(ERROR) << "Get zones failed, error: " << apache::thrift::util::enumNameSafe(code); handleErrorCode(code); onFinished(); return; } - int32_t zoneNum = zones.size(); - if (replicaFactor > zoneNum) { - LOG(ERROR) << "Replication number should less than or equal to zone number."; - handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); - onFinished(); - return; + auto zoneIter = nebula::value(zoneIterRet).get(); + while (zoneIter->valid()) { + auto zoneName = MetaKeyUtils::parseZoneName(zoneIter->key()); + zones.emplace_back(std::move(zoneName)); + zoneIter->next(); } - auto hostLoadingRet = getHostLoading(); - if (!nebula::ok(hostLoadingRet)) { - LOG(ERROR) << "Get host loading failed."; - auto retCode = nebula::error(hostLoadingRet); - if (retCode != nebula::cpp2::ErrorCode::E_LEADER_CHANGED) { - retCode = nebula::cpp2::ErrorCode::E_INVALID_PARM; + properties.set_zone_names(zones); + } else { + zones = properties.get_zone_names(); + } + + data.emplace_back(MetaKeyUtils::indexSpaceKey(spaceName), + std::string(reinterpret_cast(&spaceId), sizeof(spaceId))); + data.emplace_back(MetaKeyUtils::spaceKey(spaceId), MetaKeyUtils::spaceVal(properties)); + for (auto& zone : zones) { + auto zoneKey = MetaKeyUtils::zoneKey(zone); + auto ret = doGet(zoneKey); + if (!nebula::ok(ret)) { + auto retCode = nebula::error(ret); + if (retCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + code = nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND; } - handleErrorCode(retCode); - onFinished(); - return; + LOG(ERROR) << " Get Zone Name: " << zone << " failed."; + break; } + } - hostLoading_ = std::move(nebula::value(hostLoadingRet)); - std::unordered_map zoneHosts; - for (auto& zone : zones) { - auto zoneKey = MetaKeyUtils::zoneKey(zone); - auto zoneValueRet = doGet(std::move(zoneKey)); - if (!nebula::ok(zoneValueRet)) { - code = nebula::error(zoneValueRet); - if (code == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { - code = nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND; - } - LOG(ERROR) << "Get zone " << zone << " failed."; - break; - } + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Create space failed"; + handleErrorCode(code); + onFinished(); + return; + } - auto hosts = MetaKeyUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); - for (auto& host : hosts) { - auto hostIter = hostLoading_.find(host); - if (hostIter == hostLoading_.end()) { - hostLoading_[host] = 0; - zoneLoading_[zone] += 0; - } else { - zoneLoading_[zone] += hostIter->second; - } - } - zoneHosts[zone] = std::move(hosts); - } + int32_t zoneNum = zones.size(); + if (replicaFactor > zoneNum) { + LOG(ERROR) << "Replication number should less than or equal to zone number."; + LOG(ERROR) << "Replication number: " << replicaFactor << ", Zones size: " << zones.size(); + handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); + onFinished(); + return; + } - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Create space failed"; - handleErrorCode(code); - onFinished(); - return; + auto hostLoadingRet = getHostLoading(); + if (!nebula::ok(hostLoadingRet)) { + LOG(ERROR) << "Get host loading failed."; + auto retCode = nebula::error(hostLoadingRet); + if (retCode != nebula::cpp2::ErrorCode::E_LEADER_CHANGED) { + retCode = nebula::cpp2::ErrorCode::E_INVALID_PARM; } + handleErrorCode(retCode); + onFinished(); + return; + } - for (auto partId = 1; partId <= partitionNum; partId++) { - auto pickedZonesRet = pickLightLoadZones(replicaFactor); - if (!pickedZonesRet.ok()) { - LOG(ERROR) << "Pick zone failed."; - code = nebula::cpp2::ErrorCode::E_INVALID_PARM; - break; - } - - auto pickedZones = std::move(pickedZonesRet).value(); - auto partHostsRet = pickHostsWithZone(pickedZones, zoneHosts); - if (!partHostsRet.ok()) { - LOG(ERROR) << "Pick hosts with zone failed."; - code = nebula::cpp2::ErrorCode::E_INVALID_PARM; - break; + hostLoading_ = std::move(nebula::value(hostLoadingRet)); + std::unordered_map zoneHosts; + for (auto& zone : zones) { + auto zoneKey = MetaKeyUtils::zoneKey(zone); + auto zoneValueRet = doGet(std::move(zoneKey)); + if (!nebula::ok(zoneValueRet)) { + code = nebula::error(zoneValueRet); + if (code == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + code = nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND; } + LOG(ERROR) << "Get zone " << zone << " failed."; + break; + } - auto partHosts = std::move(partHostsRet).value(); - if (partHosts.empty()) { - LOG(ERROR) << "Pick hosts is empty."; - code = nebula::cpp2::ErrorCode::E_INVALID_PARM; - break; + auto hosts = MetaKeyUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); + for (auto& host : hosts) { + auto hostIter = hostLoading_.find(host); + if (hostIter == hostLoading_.end()) { + hostLoading_[host] = 0; + zoneLoading_[zone] += 0; + } else { + zoneLoading_[zone] += hostIter->second; } + } + zoneHosts[zone] = std::move(hosts); + } - std::stringstream ss; - for (const auto& host : partHosts) { - ss << host << ", "; - } + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Create space failed"; + handleErrorCode(code); + onFinished(); + return; + } - VLOG(3) << "Space " << spaceId << " part " << partId << " hosts " << ss.str(); - data.emplace_back(MetaKeyUtils::partKey(spaceId, partId), MetaKeyUtils::partVal(partHosts)); - } - } else { - auto hostsRet = ActiveHostsMan::getActiveHosts(kvstore_); - if (!nebula::ok(hostsRet)) { - auto retCode = nebula::error(hostsRet); - LOG(ERROR) << "Create Space Failed when get active host, error " - << apache::thrift::util::enumNameSafe(retCode); - handleErrorCode(retCode); - onFinished(); - return; + for (auto partId = 1; partId <= partitionNum; partId++) { + auto pickedZonesRet = pickLightLoadZones(replicaFactor); + if (!pickedZonesRet.ok()) { + LOG(ERROR) << "Pick zone failed."; + code = nebula::cpp2::ErrorCode::E_INVALID_PARM; + break; } - auto hosts = std::move(nebula::value(hostsRet)); - if (hosts.empty()) { - LOG(ERROR) << "Create Space Failed : No Hosts!"; - handleErrorCode(nebula::cpp2::ErrorCode::E_NO_HOSTS); - onFinished(); - return; + + auto pickedZones = std::move(pickedZonesRet).value(); + auto partHostsRet = pickHostsWithZone(pickedZones, zoneHosts); + if (!partHostsRet.ok()) { + LOG(ERROR) << "Pick hosts with zone failed."; + code = nebula::cpp2::ErrorCode::E_INVALID_PARM; + break; } - if ((int32_t)hosts.size() < replicaFactor) { - LOG(ERROR) << "Not enough hosts existed for replica " << replicaFactor << ", hosts num " - << hosts.size(); - handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); - onFinished(); - return; + auto partHosts = std::move(partHostsRet).value(); + if (partHosts.empty()) { + LOG(ERROR) << "Pick hosts is empty."; + code = nebula::cpp2::ErrorCode::E_INVALID_PARM; + break; } - for (auto partId = 1; partId <= partitionNum; partId++) { - auto partHosts = pickHosts(partId, hosts, replicaFactor); - data.emplace_back(MetaKeyUtils::partKey(spaceId, partId), MetaKeyUtils::partVal(partHosts)); + std::stringstream ss; + for (const auto& host : partHosts) { + ss << host << ", "; } + + VLOG(3) << "Space " << spaceId << " part " << partId << " hosts " << ss.str(); + data.emplace_back(MetaKeyUtils::partKey(spaceId, partId), MetaKeyUtils::partVal(partHosts)); } if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -261,17 +256,6 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { LOG(INFO) << "Create space " << spaceName; } -Hosts CreateSpaceProcessor::pickHosts(PartitionID partId, - const Hosts& hosts, - int32_t replicaFactor) { - auto startIndex = partId; - Hosts pickedHosts; - for (int32_t i = 0; i < replicaFactor; i++) { - pickedHosts.emplace_back(toThriftHost(hosts[startIndex++ % hosts.size()])); - } - return pickedHosts; -} - ErrorOr> CreateSpaceProcessor::getHostLoading() { const auto& prefix = MetaKeyUtils::partPrefix(); diff --git a/src/meta/processors/parts/CreateSpaceProcessor.h b/src/meta/processors/parts/CreateSpaceProcessor.h index 2cba00e41e5..570ab11cd74 100644 --- a/src/meta/processors/parts/CreateSpaceProcessor.h +++ b/src/meta/processors/parts/CreateSpaceProcessor.h @@ -25,8 +25,6 @@ class CreateSpaceProcessor : public BaseProcessor { explicit CreateSpaceProcessor(kvstore::KVStore* kvstore) : BaseProcessor(kvstore) {} - Hosts pickHosts(PartitionID partId, const Hosts& hosts, int32_t replicaFactor); - // Get the host with the least load in the zone StatusOr pickHostsWithZone(const std::vector& zones, const std::unordered_map& zoneHosts); diff --git a/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp b/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp index 3e9a0e4281c..654cb7aabd0 100644 --- a/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp +++ b/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp @@ -58,8 +58,7 @@ void AddHostsIntoZoneProcessor::process(const cpp2::AddHostsIntoZoneReq& req) { if (isNew) { // If you are creating a new zone, should make sure the zone not existed. if (nebula::ok(zoneValueRet)) { - LOG(ERROR) << "Zone " << zoneName - << " have existed error: " << apache::thrift::util::enumNameSafe(code); + LOG(ERROR) << "Zone " << zoneName << " have existed"; handleErrorCode(nebula::cpp2::ErrorCode::E_EXISTED); onFinished(); return; diff --git a/src/meta/processors/zone/AddHostsProcessor.cpp b/src/meta/processors/zone/AddHostsProcessor.cpp index 56afe5d9b3a..7672413a6e2 100644 --- a/src/meta/processors/zone/AddHostsProcessor.cpp +++ b/src/meta/processors/zone/AddHostsProcessor.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. + * This source code is licensed under Apache 2.0 License. */ #include "meta/processors/zone/AddHostsProcessor.h" @@ -32,7 +31,7 @@ void AddHostsProcessor::process(const cpp2::AddHostsReq& req) { } std::vector data; - nebula::cpp2::ErrorCode code; + nebula::cpp2::ErrorCode code = nebula::cpp2::ErrorCode::SUCCEEDED; for (auto& host : hosts) { // Ensure that the node is not registered. auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); diff --git a/src/meta/processors/zone/AddHostsProcessor.h b/src/meta/processors/zone/AddHostsProcessor.h index 5315aa6d5b1..0e43023ae79 100644 --- a/src/meta/processors/zone/AddHostsProcessor.h +++ b/src/meta/processors/zone/AddHostsProcessor.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. + * This source code is licensed under Apache 2.0 License. */ #ifndef META_ADDHOSTSPROCESSOR_H diff --git a/src/meta/processors/zone/DropHostsProcessor.cpp b/src/meta/processors/zone/DropHostsProcessor.cpp index a3541cccf59..12802889906 100644 --- a/src/meta/processors/zone/DropHostsProcessor.cpp +++ b/src/meta/processors/zone/DropHostsProcessor.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. + * This source code is licensed under Apache 2.0 License. */ #include "meta/processors/zone/DropHostsProcessor.h" diff --git a/src/meta/processors/zone/DropHostsProcessor.h b/src/meta/processors/zone/DropHostsProcessor.h index a0bec3b27fd..41057636c02 100644 --- a/src/meta/processors/zone/DropHostsProcessor.h +++ b/src/meta/processors/zone/DropHostsProcessor.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. + * This source code is licensed under Apache 2.0 License. */ #ifndef META_DROPHOSTSPROCESSOR_H diff --git a/src/meta/processors/zone/MergeZoneProcessor.cpp b/src/meta/processors/zone/MergeZoneProcessor.cpp new file mode 100644 index 00000000000..c8d3b011942 --- /dev/null +++ b/src/meta/processors/zone/MergeZoneProcessor.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "meta/processors/zone/MergeZoneProcessor.h" + +namespace nebula { +namespace meta { + +void MergeZoneProcessor::process(const cpp2::MergeZoneReq& req) { UNUSED(req); } + +} // namespace meta +} // namespace nebula diff --git a/src/meta/processors/zone/MergeZoneProcessor.h b/src/meta/processors/zone/MergeZoneProcessor.h new file mode 100644 index 00000000000..ae59e667247 --- /dev/null +++ b/src/meta/processors/zone/MergeZoneProcessor.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef META_MERGEZONEPROCESSOR_H +#define META_MERGEZONEPROCESSOR_H + +#include "meta/processors/BaseProcessor.h" + +namespace nebula { +namespace meta { + +class MergeZoneProcessor : public BaseProcessor { + public: + static MergeZoneProcessor* instance(kvstore::KVStore* kvstore) { + return new MergeZoneProcessor(kvstore); + } + + void process(const cpp2::MergeZoneReq& req); + + private: + explicit MergeZoneProcessor(kvstore::KVStore* kvstore) : BaseProcessor(kvstore) {} +}; + +} // namespace meta +} // namespace nebula + +#endif // META_MERGEZONEPROCESSOR_H diff --git a/src/meta/processors/zone/SplitZoneProcessor.cpp b/src/meta/processors/zone/SplitZoneProcessor.cpp new file mode 100644 index 00000000000..8205e83d34e --- /dev/null +++ b/src/meta/processors/zone/SplitZoneProcessor.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "meta/processors/zone/SplitZoneProcessor.h" + +namespace nebula { +namespace meta { + +void SplitZoneProcessor::process(const cpp2::SplitZoneReq& req) { UNUSED(req); } + +} // namespace meta +} // namespace nebula diff --git a/src/meta/processors/zone/SplitZoneProcessor.h b/src/meta/processors/zone/SplitZoneProcessor.h new file mode 100644 index 00000000000..f69abe84143 --- /dev/null +++ b/src/meta/processors/zone/SplitZoneProcessor.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef META_SPLITZONEPROCESSOR_H +#define META_SPLITZONEPROCESSOR_H + +#include "meta/processors/BaseProcessor.h" + +namespace nebula { +namespace meta { + +class SplitZoneProcessor : public BaseProcessor { + public: + static SplitZoneProcessor* instance(kvstore::KVStore* kvstore) { + return new SplitZoneProcessor(kvstore); + } + + void process(const cpp2::SplitZoneReq& req); + + private: + explicit SplitZoneProcessor(kvstore::KVStore* kvstore) : BaseProcessor(kvstore) {} +}; + +} // namespace meta +} // namespace nebula + +#endif // META_SPLITZONEPROCESSOR_H diff --git a/src/meta/test/AuthProcessorTest.cpp b/src/meta/test/AuthProcessorTest.cpp index 3c0d95493d4..f1a7a7a32bc 100644 --- a/src/meta/test/AuthProcessorTest.cpp +++ b/src/meta/test/AuthProcessorTest.cpp @@ -10,6 +10,7 @@ #include "meta/processors/parts/CreateSpaceProcessor.h" #include "meta/processors/parts/DropSpaceProcessor.h" #include "meta/processors/user/AuthenticationProcessor.h" +#include "meta/processors/zone/AddHostsProcessor.h" #include "meta/test/TestUtils.h" namespace nebula { @@ -161,7 +162,16 @@ TEST(AuthProcessorTest, DropUserTest) { TEST(AuthProcessorTest, GrantRevokeTest) { fs::TempDir rootPath("/tmp/GrantRevokeTest.XXXXXX"); std::unique_ptr kv(MockCluster::initMetaKV(rootPath.path())); - TestUtils::createSomeHosts(kv.get()); + { + cpp2::AddHostsReq req; + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } GraphSpaceID space1, space2; // create space1 { diff --git a/src/meta/test/IndexProcessorTest.cpp b/src/meta/test/IndexProcessorTest.cpp index 57a4c5627e9..89d82d6ba78 100644 --- a/src/meta/test/IndexProcessorTest.cpp +++ b/src/meta/test/IndexProcessorTest.cpp @@ -23,6 +23,7 @@ #include "meta/processors/schema/CreateTagProcessor.h" #include "meta/processors/schema/DropEdgeProcessor.h" #include "meta/processors/schema/DropTagProcessor.h" +#include "meta/processors/zone/AddHostsProcessor.h" #include "meta/test/TestUtils.h" namespace nebula { @@ -2036,8 +2037,16 @@ TEST(IndexProcessorTest, AlterWithFTIndexTest) { TEST(ProcessorTest, IndexIdInSpaceRangeTest) { fs::TempDir rootPath("/tmp/IndexIdInSpaceRangeTest.XXXXXX"); auto kv = MockCluster::initMetaKV(rootPath.path()); - TestUtils::createSomeHosts(kv.get()); - + { + cpp2::AddHostsReq req; + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } // mock one space and ten tag, ten edge { // space Id is 1 diff --git a/src/meta/test/MetaClientTest.cpp b/src/meta/test/MetaClientTest.cpp index dcf477d2e91..ef563bfed78 100644 --- a/src/meta/test/MetaClientTest.cpp +++ b/src/meta/test/MetaClientTest.cpp @@ -17,6 +17,7 @@ #include "common/network/NetworkUtils.h" #include "common/utils/MetaKeyUtils.h" #include "interface/gen-cpp2/common_constants.h" +#include "meta/processors/zone/AddHostsProcessor.h" #include "meta/test/TestUtils.h" #include "mock/MockCluster.h" @@ -39,11 +40,14 @@ TEST(MetaClientTest, InterfacesTest) { cluster.startMeta(rootPath.path()); auto options = meta::MetaClientOptions(); cluster.initMetaClient(); - auto* kv = cluster.metaKV_.get(); auto* client = cluster.metaClient_.get(); GraphSpaceID spaceId = 0; - TestUtils::createSomeHosts(kv); + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } { // Test createSpace, listSpaces, getPartsAlloc. { @@ -348,10 +352,12 @@ TEST(MetaClientTest, TagTest) { mock::MockCluster cluster; cluster.startMeta(rootPath.path()); cluster.initMetaClient(); - auto* kv = cluster.metaKV_.get(); auto* client = cluster.metaClient_.get(); - - TestUtils::createSomeHosts(kv); + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } meta::cpp2::SpaceDesc spaceDesc; spaceDesc.set_space_name("default"); spaceDesc.set_partition_num(9); @@ -581,10 +587,12 @@ TEST(MetaClientTest, EdgeTest) { mock::MockCluster cluster; cluster.startMeta(rootPath.path()); cluster.initMetaClient(); - auto* kv = cluster.metaKV_.get(); auto* client = cluster.metaClient_.get(); - - TestUtils::createSomeHosts(kv); + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } meta::cpp2::SpaceDesc spaceDesc; spaceDesc.set_space_name("default_space"); spaceDesc.set_partition_num(9); @@ -688,10 +696,13 @@ TEST(MetaClientTest, TagIndexTest) { mock::MockCluster cluster; cluster.startMeta(rootPath.path()); cluster.initMetaClient(); - auto* kv = cluster.metaKV_.get(); auto* client = cluster.metaClient_.get(); + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } - TestUtils::createSomeHosts(kv); meta::cpp2::SpaceDesc spaceDesc; spaceDesc.set_space_name("default_space"); spaceDesc.set_partition_num(8); @@ -864,10 +875,13 @@ TEST(MetaClientTest, EdgeIndexTest) { mock::MockCluster cluster; cluster.startMeta(rootPath.path()); cluster.initMetaClient(); - auto* kv = cluster.metaKV_.get(); auto* client = cluster.metaClient_.get(); + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } - TestUtils::createSomeHosts(kv); meta::cpp2::SpaceDesc spaceDesc; spaceDesc.set_space_name("default_space"); spaceDesc.set_partition_num(8); @@ -1173,11 +1187,23 @@ TEST(MetaClientTest, DiffTest) { mock::MockCluster cluster; cluster.startMeta(rootPath.path()); auto* kv = cluster.metaKV_.get(); - cluster.initMetaClient(); - auto* client = cluster.metaClient_.get(); std::vector hosts = {{"0", 0}}; - TestUtils::createSomeHosts(kv, std::move(hosts)); + { + cpp2::AddHostsReq req; + req.set_hosts(hosts); + auto* processor = AddHostsProcessor::instance(kv); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + meta::MetaClientOptions options; + options.localHost_ = {"0", 0}; + options.role_ = meta::cpp2::HostRole::STORAGE; + options.clusterId_ = 10; + cluster.initMetaClient(options); + auto* client = cluster.metaClient_.get(); auto listener = std::make_unique(); client->registerListener(listener.get()); { @@ -1192,6 +1218,7 @@ TEST(MetaClientTest, DiffTest) { ASSERT_EQ(hosts[i], hostAddr); } } + sleep(FLAGS_heartbeat_interval_secs + 1); { // Test Create Space and List Spaces meta::cpp2::SpaceDesc spaceDesc; @@ -1233,8 +1260,6 @@ TEST(MetaClientTest, ListenerDiffTest) { cluster.startMeta(rootPath.path()); cluster.initMetaClient(); auto* kv = cluster.metaKV_.get(); - std::vector hosts = {{"0", 0}}; - TestUtils::createSomeHosts(kv, std::move(hosts)); auto* console = cluster.metaClient_.get(); auto testListener = std::make_unique(); console->registerListener(testListener.get()); @@ -1247,6 +1272,11 @@ TEST(MetaClientTest, ListenerDiffTest) { auto threadPool = std::make_shared(1); auto metaAddrs = {HostAddr(cluster.localIP(), cluster.metaServer_->port_)}; auto client = std::make_unique(threadPool, metaAddrs, options); + { + std::vector hosts = {{"0", 0}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } client->waitForMetadReady(); auto listener = std::make_unique(); @@ -1654,9 +1684,13 @@ TEST(MetaClientTest, ListenerTest) { auto threadPool = std::make_shared(1); auto localhosts = std::vector{HostAddr(localIp, localMetaPort)}; auto client = std::make_shared(threadPool, localhosts); + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } client->waitForMetadReady(); - TestUtils::createSomeHosts(kv); meta::cpp2::SpaceDesc spaceDesc; spaceDesc.set_space_name("default"); spaceDesc.set_partition_num(9); @@ -1732,79 +1766,11 @@ TEST(MetaClientTest, VerifyClientTest) { } FLAGS_enable_client_white_list = false; } - -TEST(MetaClientTest, RocksdbOptionsTest) { - FLAGS_heartbeat_interval_secs = 1; - fs::TempDir rootPath("/tmp/RocksdbOptionsTest.XXXXXX"); - - mock::MockCluster cluster; - cluster.startMeta(rootPath.path(), HostAddr("127.0.0.1", 0)); - auto* kv = cluster.metaKV_.get(); - TestUtils::createSomeHosts(kv, {{"0", 0}}); - - MetaClientOptions options; - // Now the `--local_config' option only affect if initialize the configuration - // from meta not affect `show/update/get configs' - options.skipConfig_ = false; - - cluster.initMetaClient(std::move(options)); - auto* client = cluster.metaClient_.get(); - - auto listener = std::make_unique(); - auto module = cpp2::ConfigModule::STORAGE; - auto mode = meta::cpp2::ConfigMode::MUTABLE; - - client->registerListener(listener.get()); - client->gflagsModule_ = module; - - // mock some rocksdb gflags to meta - { - auto name = "rocksdb_db_options"; - std::vector configItems; - FLAGS_rocksdb_db_options = - R"({"disable_auto_compactions":"false","write_buffer_size":"1048576"})"; - Map map; - map.kvs.emplace("disable_auto_compactions", "false"); - map.kvs.emplace("write_buffer_size", "1048576"); - configItems.emplace_back(initConfigItem(module, name, mode, Value(map))); - client->regConfig(configItems); - } - { - std::vector hosts = {{"0", 0}}; - TestUtils::registerHB(cluster.metaKV_.get(), hosts); - meta::cpp2::SpaceDesc spaceDesc; - spaceDesc.set_space_name("default_space"); - spaceDesc.set_partition_num(9); - spaceDesc.set_replica_factor(1); - client->createSpace(spaceDesc).get(); - sleep(FLAGS_heartbeat_interval_secs + 1); - } - { - std::string name = "rocksdb_db_options"; - Map map; - map.kvs.emplace("disable_auto_compactions", "true"); - map.kvs.emplace("level0_file_num_compaction_trigger", "4"); - - // update config - auto setRet = client->setConfig(module, name, Value(map)).get(); - ASSERT_TRUE(setRet.ok()); - - // get from meta server - auto getRet = client->getConfig(module, name).get(); - ASSERT_TRUE(getRet.ok()); - auto item = getRet.value().front(); - - sleep(FLAGS_heartbeat_interval_secs + 1); - ASSERT_EQ(listener->options["disable_auto_compactions"], "\"true\""); - ASSERT_EQ(listener->options["level0_file_num_compaction_trigger"], "\"4\""); - } -} - TEST(MetaClientTest, HostsTest) { FLAGS_heartbeat_interval_secs = 1; fs::TempDir rootPath("/tmp/HostsTest.XXXXXX"); mock::MockCluster cluster; - cluster.startMeta(rootPath.path(), HostAddr("127.0.0.1", 0)); + cluster.startMeta(rootPath.path()); cluster.initMetaClient(); auto* client = cluster.metaClient_.get(); { @@ -1870,6 +1836,11 @@ TEST(MetaClientTest, AddHostsIntoNewZoneTest) { cluster.startMeta(rootPath.path(), HostAddr("127.0.0.1", 0)); cluster.initMetaClient(); auto* client = cluster.metaClient_.get(); + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } { // Add host into zone with duplicate hosts std::vector hosts = {{"127.0.0.1", 8988}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; @@ -2254,6 +2225,79 @@ TEST(MetaClientTest, RenameZoneTest) { } } +TEST(MetaClientTest, RocksdbOptionsTest) { + FLAGS_heartbeat_interval_secs = 1; + fs::TempDir rootPath("/tmp/RocksdbOptionsTest.XXXXXX"); + + mock::MockCluster cluster; + cluster.startMeta(rootPath.path()); + // auto* kv = cluster.metaKV_.get(); + // TestUtils::createSomeHosts(kv, {{"0", 0}}); + + MetaClientOptions options; + // Now the `--local_config' option only affect if initialize the configuration + // from meta not affect `show/update/get configs' + options.skipConfig_ = false; + + cluster.initMetaClient(std::move(options)); + auto* client = cluster.metaClient_.get(); + { + // Add single host + std::vector hosts = {{"0", 0}}; + auto result = client->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } + + auto listener = std::make_unique(); + auto module = cpp2::ConfigModule::STORAGE; + auto mode = meta::cpp2::ConfigMode::MUTABLE; + + client->registerListener(listener.get()); + client->gflagsModule_ = module; + + // mock some rocksdb gflags to meta + { + auto name = "rocksdb_db_options"; + std::vector configItems; + FLAGS_rocksdb_db_options = + R"({"disable_auto_compactions":"false","write_buffer_size":"1048576"})"; + Map map; + map.kvs.emplace("disable_auto_compactions", "false"); + map.kvs.emplace("write_buffer_size", "1048576"); + configItems.emplace_back(initConfigItem(module, name, mode, Value(map))); + client->regConfig(configItems); + } + { + std::vector hosts = {{"0", 0}}; + TestUtils::registerHB(cluster.metaKV_.get(), hosts); + meta::cpp2::SpaceDesc spaceDesc; + spaceDesc.set_space_name("default_space"); + spaceDesc.set_partition_num(9); + spaceDesc.set_replica_factor(1); + client->createSpace(spaceDesc).get(); + sleep(FLAGS_heartbeat_interval_secs + 1); + } + { + std::string name = "rocksdb_db_options"; + Map map; + map.kvs.emplace("disable_auto_compactions", "true"); + map.kvs.emplace("level0_file_num_compaction_trigger", "4"); + + // update config + auto setRet = client->setConfig(module, name, Value(map)).get(); + ASSERT_TRUE(setRet.ok()); + + // get from meta server + auto getRet = client->getConfig(module, name).get(); + ASSERT_TRUE(getRet.ok()); + auto item = getRet.value().front(); + + sleep(FLAGS_heartbeat_interval_secs + 1); + ASSERT_EQ(listener->options["disable_auto_compactions"], "\"true\""); + ASSERT_EQ(listener->options["level0_file_num_compaction_trigger"], "\"4\""); + } +} + } // namespace meta } // namespace nebula diff --git a/src/meta/test/ProcessorTest.cpp b/src/meta/test/ProcessorTest.cpp index f9e2662378d..81cffadd621 100644 --- a/src/meta/test/ProcessorTest.cpp +++ b/src/meta/test/ProcessorTest.cpp @@ -310,8 +310,17 @@ TEST(ProcessorTest, HashTest) { TEST(ProcessorTest, SpaceTest) { fs::TempDir rootPath("/tmp/SpaceTest.XXXXXX"); auto kv = MockCluster::initMetaKV(rootPath.path()); - auto hostsNum = TestUtils::createSomeHosts(kv.get()); - + { + cpp2::AddHostsReq req; + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + int32_t hostsNum = 4; { cpp2::SpaceDesc properties; properties.set_space_name("default_space"); @@ -467,8 +476,16 @@ TEST(ProcessorTest, SpaceTest) { TEST(ProcessorTest, CreateTagTest) { fs::TempDir rootPath("/tmp/CreateTagTest.XXXXXX"); auto kv = MockCluster::initMetaKV(rootPath.path()); - TestUtils::createSomeHosts(kv.get()); - + { + cpp2::AddHostsReq req; + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } { cpp2::SpaceDesc properties; properties.set_space_name("first_space"); @@ -662,8 +679,16 @@ TEST(ProcessorTest, CreateTagTest) { TEST(ProcessorTest, CreateEdgeTest) { fs::TempDir rootPath("/tmp/CreateEdgeTest.XXXXXX"); auto kv = MockCluster::initMetaKV(rootPath.path()); - TestUtils::createSomeHosts(kv.get()); - + { + cpp2::AddHostsReq req; + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } { cpp2::SpaceDesc properties; properties.set_space_name("default_space"); @@ -846,8 +871,16 @@ TEST(ProcessorTest, CreateEdgeTest) { TEST(ProcessorTest, KVOperationTest) { fs::TempDir rootPath("/tmp/KVOperationTest.XXXXXX"); auto kv = MockCluster::initMetaKV(rootPath.path()); - TestUtils::createSomeHosts(kv.get()); - + { + cpp2::AddHostsReq req; + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } { cpp2::SpaceDesc properties; properties.set_space_name("default_space"); @@ -2239,8 +2272,16 @@ TEST(ProcessorTest, AlterEdgeTest) { TEST(ProcessorTest, SameNameTagsTest) { fs::TempDir rootPath("/tmp/SameNameTagsTest.XXXXXX"); auto kv = MockCluster::initMetaKV(rootPath.path()); - TestUtils::createSomeHosts(kv.get()); - + { + cpp2::AddHostsReq req; + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } { cpp2::SpaceDesc properties; properties.set_space_name("default_space"); @@ -2479,8 +2520,16 @@ TEST(ProcessorTest, SessionManagerTest) { TEST(ProcessorTest, TagIdAndEdgeTypeInSpaceRangeTest) { fs::TempDir rootPath("/tmp/TagIdAndEdgeTypeInSpaceRangeTest.XXXXXX"); auto kv = MockCluster::initMetaKV(rootPath.path()); - TestUtils::createSomeHosts(kv.get()); - + { + cpp2::AddHostsReq req; + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + req.set_hosts(std::move(hosts)); + auto* processor = AddHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } // mock one space and ten tag, ten edge { // space Id is 1 diff --git a/src/meta/test/TestUtils.h b/src/meta/test/TestUtils.h index 4d75c5382f2..2440486c3e1 100644 --- a/src/meta/test/TestUtils.h +++ b/src/meta/test/TestUtils.h @@ -86,6 +86,7 @@ class TestUtils { // Record machine information std::vector machines; for (auto& host : hosts) { + VLOG(3) << "Registe machine: " << host; machines.emplace_back(nebula::MetaKeyUtils::machineKey(host.host, host.port), ""); } folly::Baton baton; diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 061d8993f2b..5206c6d2d67 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -358,7 +358,7 @@ static constexpr size_t kCommentLengthLimit = 256; %type rebuild_tag_index_sentence rebuild_edge_index_sentence rebuild_fulltext_index_sentence %type add_hosts_sentence drop_hosts_sentence %type drop_zone_sentence desc_zone_sentence -%type merge_zone_sentence split_zone_sentence rename_zone_sentence +%type merge_zone_sentence /*split_zone_sentence*/ rename_zone_sentence %type create_snapshot_sentence drop_snapshot_sentence %type add_listener_sentence remove_listener_sentence list_listener_sentence @@ -2713,11 +2713,11 @@ drop_zone_sentence } ; -split_zone_sentence - : KW_SPLIT KW_ZONE name_label KW_FROM zone_name_list { - $$ = new SplitZoneSentence($3, $5); - } - ; +// split_zone_sentence +// : KW_SPLIT KW_ZONE name_label KW_FROM zone_name_list { +// $$ = new SplitZoneSentence($3, $5); +// } +// ; rename_zone_sentence : KW_RENAME KW_ZONE name_label KW_TO name_label { @@ -3679,7 +3679,7 @@ maintain_sentence | drop_hosts_sentence { $$ = $1; } | merge_zone_sentence { $$ = $1; } | drop_zone_sentence { $$ = $1; } - | split_zone_sentence { $$ = $1; } + // | split_zone_sentence { $$ = $1; } | rename_zone_sentence { $$ = $1; } | desc_zone_sentence { $$ = $1; } | show_sentence { $$ = $1; } diff --git a/src/parser/scanner.lex b/src/parser/scanner.lex index 1a4e4a0a033..4d92d09637b 100644 --- a/src/parser/scanner.lex +++ b/src/parser/scanner.lex @@ -269,7 +269,6 @@ CHINESE_LABEL ({U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U})+ "LINESTRING" { return TokenType::KW_LINESTRING; } "POLYGON" { return TokenType::KW_POLYGON; } "MERGE" { return TokenType::KW_MERGE; } -"SPLIT" { return TokenType::KW_SPLIT; } "RENAME" { return TokenType::KW_RENAME; } "TRUE" { yylval->boolval = true; return TokenType::BOOL; } diff --git a/src/parser/test/ScannerTest.cpp b/src/parser/test/ScannerTest.cpp index 66b7a3cf16a..f67a3265aa8 100644 --- a/src/parser/test/ScannerTest.cpp +++ b/src/parser/test/ScannerTest.cpp @@ -506,9 +506,6 @@ TEST(Scanner, Basic) { CHECK_SEMANTIC_TYPE("MERGE", TokenType::KW_MERGE), CHECK_SEMANTIC_TYPE("Merge", TokenType::KW_MERGE), CHECK_SEMANTIC_TYPE("Merge", TokenType::KW_MERGE), - CHECK_SEMANTIC_TYPE("SPLIT", TokenType::KW_SPLIT), - CHECK_SEMANTIC_TYPE("Split", TokenType::KW_SPLIT), - CHECK_SEMANTIC_TYPE("split", TokenType::KW_SPLIT), CHECK_SEMANTIC_TYPE("RENAME", TokenType::KW_RENAME), CHECK_SEMANTIC_TYPE("Rename", TokenType::KW_RENAME), CHECK_SEMANTIC_TYPE("rename", TokenType::KW_RENAME), diff --git a/src/storage/test/KVClientTest.cpp b/src/storage/test/KVClientTest.cpp index 9be13c4a762..e52c9451e95 100644 --- a/src/storage/test/KVClientTest.cpp +++ b/src/storage/test/KVClientTest.cpp @@ -41,20 +41,18 @@ TEST(KVClientTest, SimpleTest) { HostAddr storageAddr{storageName, storagePort}; cluster.startMeta(metaPath.path()); - folly::Baton baton; - auto* kv = cluster.metaKV_.get(); - std::vector machines; - machines.emplace_back(nebula::MetaKeyUtils::machineKey(storageName, storagePort), ""); - kv->asyncMultiPut( - kDefaultSpaceId, kDefaultPartId, std::move(machines), [&](auto) { baton.post(); }); - baton.wait(); - meta::MetaClientOptions options; options.localHost_ = storageAddr; - options.role_ = meta::cpp2::HostRole::STORAGE; cluster.initMetaClient(options); - cluster.startStorage(storageAddr, storagePath.path()); + auto* metaClient = cluster.metaClient_.get(); + { + LOG(INFO) << "registed " << storageAddr; + std::vector hosts = {storageAddr}; + auto result = metaClient->addHosts(std::move(hosts)).get(); + EXPECT_TRUE(result.ok()); + } + cluster.startStorage(storageAddr, storagePath.path()); auto client = cluster.initGraphStorageClient(); // kv interface test { diff --git a/tests/admin/test_space.py b/tests/admin/test_space.py index 028ea0456e1..917bdff2566 100644 --- a/tests/admin/test_space.py +++ b/tests/admin/test_space.py @@ -20,13 +20,10 @@ def test_space(self): resp = self.client.execute('CREATE SPACE space_with_default_options (vid_type=FIXED_STRING(8))') self.check_resp_succeeded(resp) - resp = self.client.execute('CREATE SPACE space_on_default_group on default') - self.check_resp_failed(resp) - # check result resp = self.client.execute('DESC SPACE space_with_default_options') expect_result = [['space_with_default_options', 100, 1, 'utf8', 'utf8_bin', - 'FIXED_STRING(8)', False, 'default', T_EMPTY]] + 'FIXED_STRING(8)', False, 'default_zone', T_EMPTY]] self.check_result(resp, expect_result, {0}) # drop space @@ -46,7 +43,7 @@ def test_space(self): resp = self.client.execute('DESC SPACE default_space') self.check_resp_succeeded(resp) expect_result = [['default_space', 9, 1, 'utf8', 'utf8_bin', 'FIXED_STRING(8)', - False, 'default', T_EMPTY]] + False, 'default_zone', T_EMPTY]] self.check_result(resp, expect_result, {0}) # show create space @@ -61,7 +58,7 @@ def test_space(self): 'collate = utf8_bin, '\ 'vid_type = FIXED_STRING(8), '\ 'atomic_edge = false) '\ - 'ON default' + 'ON default_zone' expect_result = [['default_space', create_space_str_result]] self.check_result(resp, expect_result) @@ -95,7 +92,7 @@ def test_charset_collate(self): resp = self.client.execute('DESC SPACE space_charset_collate') self.check_resp_succeeded(resp) - expect_result = [['space_charset_collate', 9, 1, 'utf8', 'utf8_bin', 'FIXED_STRING(8)', False, 'default', T_EMPTY]] + expect_result = [['space_charset_collate', 9, 1, 'utf8', 'utf8_bin', 'FIXED_STRING(8)', False, 'default_zone', T_EMPTY]] self.check_result(resp, expect_result, {0}) # drop space @@ -108,7 +105,7 @@ def test_charset_collate(self): resp = self.client.execute('DESC SPACE space_charset') self.check_resp_succeeded(resp) - expect_result = [['space_charset', 9, 1, 'utf8', 'utf8_bin', 'FIXED_STRING(8)', False, 'default', T_EMPTY]] + expect_result = [['space_charset', 9, 1, 'utf8', 'utf8_bin', 'FIXED_STRING(8)', False, 'default_zone', T_EMPTY]] self.check_result(resp, expect_result, {0}) # drop space @@ -121,7 +118,7 @@ def test_charset_collate(self): resp = self.client.execute('DESC SPACE space_collate') self.check_resp_succeeded(resp) - expect_result = [['space_collate', 9, 1, 'utf8', 'utf8_bin', 'FIXED_STRING(8)', False, 'default', T_EMPTY]] + expect_result = [['space_collate', 9, 1, 'utf8', 'utf8_bin', 'FIXED_STRING(8)', False, 'default_zone', T_EMPTY]] self.check_result(resp, expect_result, {0}) # drop space @@ -159,7 +156,7 @@ def test_charset_collate(self): resp = self.client.execute('DESC SPACE space_capital') self.check_resp_succeeded(resp) expect_result = [['space_capital', 9, 1, 'utf8', 'utf8_bin', 'FIXED_STRING(8)', - False, 'default', T_EMPTY]] + False, 'default_zone', T_EMPTY]] self.check_result(resp, expect_result, {0}) # drop space @@ -211,7 +208,7 @@ def test_create_space_with_string_vid(self): resp = self.client.execute('DESC SPACE space_string_vid') self.check_resp_succeeded(resp) - expect_result = [['space_string_vid', 9, 1, 'utf8', 'utf8_bin', 'FIXED_STRING(30)', False, 'default', T_EMPTY]] + expect_result = [['space_string_vid', 9, 1, 'utf8', 'utf8_bin', 'FIXED_STRING(30)', False, 'default_zone', T_EMPTY]] self.check_result(resp, expect_result, {0}) # clean up @@ -226,7 +223,7 @@ def test_create_space_with_int_vid(self): resp = self.client.execute('DESC SPACE space_int_vid') self.check_resp_succeeded(resp) - expect_result = [['space_int_vid', 9, 1, 'utf8', 'utf8_bin', 'INT64', False, 'default', T_EMPTY]] + expect_result = [['space_int_vid', 9, 1, 'utf8', 'utf8_bin', 'INT64', False, 'default_zone', T_EMPTY]] self.check_result(resp, expect_result, {0}) # clean up diff --git a/tests/common/nebula_service.py b/tests/common/nebula_service.py index c5d7ae8faff..502e79a38e2 100644 --- a/tests/common/nebula_service.py +++ b/tests/common/nebula_service.py @@ -18,6 +18,8 @@ from contextlib import closing from tests.common.constants import TMP_DIR +from nebula2.gclient.net import ConnectionPool +from nebula2.Config import Config NEBULA_START_COMMAND_FORMAT = "bin/nebula-{} --flagfile conf/nebula-{}.conf {}" @@ -128,6 +130,8 @@ def __init__( self.all_processes = [] self.all_ports = [] self.metad_param, self.storaged_param, self.graphd_param = {}, {}, {} + self.storaged_port = 0 + self.graphd_port = 0 self.ca_signed = ca_signed self.debug_log = debug_log self.ports_per_process = 4 @@ -161,6 +165,8 @@ def init_process(self): ) self.storaged_processes.append(storaged) index += self.ports_per_process + if suffix_index == 0: + self.storaged_port = self.all_ports[0] for suffix_index in range(self.graphd_num): graphd = NebulaProcess( @@ -171,6 +177,8 @@ def init_process(self): ) self.graphd_processes.append(graphd) index += self.ports_per_process + if suffix_index == 0: + self.graphd_port = self.all_ports[0] self.all_processes = ( self.metad_processes + self.storaged_processes + self.graphd_processes @@ -349,6 +357,23 @@ def start(self): for p in self.all_processes: p.start() + time.sleep(3) + config = Config() + config.max_connection_pool_size = 20 + config.timeout = 60000 + # init connection pool + client_pool = ConnectionPool() + # assert client_pool.init([("127.0.0.1", int(self.graphd_port))], config) + assert client_pool.init([("127.0.0.1", self.graphd_processes[0].tcp_port)], config) + + cmd = "ADD HOSTS 127.0.0.1:" + str(self.storaged_processes[0].tcp_port) + " INTO NEW ZONE default_zone" + print(cmd) + + # get session from the pool + client = client_pool.get_session('root', 'nebula') + resp = client.execute(cmd) + client.release() + # wait nebula start server_ports = [p.tcp_port for p in self.all_processes] if not self._check_servers_status(server_ports): diff --git a/tests/maintain/test_zone.py b/tests/maintain/test_zone.py index 9e74d876af9..74128fe3110 100644 --- a/tests/maintain/test_zone.py +++ b/tests/maintain/test_zone.py @@ -10,36 +10,34 @@ class TestZone(NebulaTestSuite): def test_zone(self): - resp = self.client.execute('SHOW HOSTS') - self.check_resp_succeeded(resp) - assert not resp.is_empty() - storage_port = resp.row_values(0)[1].as_int() - # Add Zone - resp = self.client.execute('ADD ZONE zone_0 127.0.0.1:' + str(storage_port)) - self.check_resp_succeeded(resp) - - # Get Zone - resp = self.client.execute('DESC ZONE zone_0') - self.check_resp_succeeded(resp) - - resp = self.client.execute('DESCRIBE ZONE zone_0') - self.check_resp_succeeded(resp) - - # Get Zone which is not exist - resp = self.client.execute('DESC ZONE zone_not_exist') - self.check_resp_failed(resp) - - resp = self.client.execute('DESCRIBE ZONE zone_not_exist') - self.check_resp_failed(resp) - - # SHOW Zones - resp = self.client.execute('SHOW ZONES') - self.check_resp_succeeded(resp) - - # Drop Zone - resp = self.client.execute('DROP ZONE zone_0') - self.check_resp_succeeded(resp) - - # Drop Zone which is not exist - resp = self.client.execute('DROP ZONE zone_0') - self.check_resp_failed(resp) + pass + # resp = self.client.execute('SHOW HOSTS') + # self.check_resp_succeeded(resp) + # assert not resp.is_empty() + # storage_port = resp.row_values(0)[1].as_int() + + # # Get Zone + # resp = self.client.execute('DESC ZONE default_zone_127.0.0.1_' + str(storage_port)) + # self.check_resp_succeeded(resp) + + # resp = self.client.execute('DESCRIBE ZONE zone_0') + # self.check_resp_succeeded(resp) + + # # Get Zone which is not exist + # resp = self.client.execute('DESC ZONE zone_not_exist') + # self.check_resp_failed(resp) + + # resp = self.client.execute('DESCRIBE ZONE zone_not_exist') + # self.check_resp_failed(resp) + + # # SHOW Zones + # resp = self.client.execute('SHOW ZONES') + # self.check_resp_succeeded(resp) + + # # Drop Zone + # resp = self.client.execute('DROP ZONE zone_0') + # self.check_resp_succeeded(resp) + + # # Drop Zone which is not exist + # resp = self.client.execute('DROP ZONE zone_0') + # self.check_resp_failed(resp) diff --git a/tests/tck/features/schema/Comment.feature b/tests/tck/features/schema/Comment.feature index f877007f633..638478bed77 100644 --- a/tests/tck/features/schema/Comment.feature +++ b/tests/tck/features/schema/Comment.feature @@ -15,15 +15,15 @@ Feature: Schema Comment SHOW CREATE SPACE ; """ Then the result should be, in any order: - | Space | Create Space | - | "" | "CREATE SPACE `` (partition_num = 100, replica_factor = 1, charset = utf8, collate = utf8_bin, vid_type = FIXED_STRING(8), atomic_edge = false) ON default comment = ''" | + | Space | Create Space | + | "" | "CREATE SPACE `` (partition_num = 100, replica_factor = 1, charset = utf8, collate = utf8_bin, vid_type = FIXED_STRING(8), atomic_edge = false) ON default_zone comment = ''" | When executing query: """ DESC SPACE ; """ Then the result should be, in any order: - | ID | Name | Partition Number | Replica Factor | Charset | Collate | Vid Type | Atomic Edge | Group | Comment | - | /\d+/ | "" | 100 | 1 | "utf8" | "utf8_bin" | "FIXED_STRING(8)" | false | "default" | "" | + | ID | Name | Partition Number | Replica Factor | Charset | Collate | Vid Type | Atomic Edge | Zones | Comment | + | /\d+/ | "" | 100 | 1 | "utf8" | "utf8_bin" | "FIXED_STRING(8)" | false | "default_zone" | "" | When executing query: """ DROP SPACE ; @@ -47,15 +47,15 @@ Feature: Schema Comment SHOW CREATE SPACE test_comment_not_set; """ Then the result should be, in any order: - | Space | Create Space | - | "test_comment_not_set" | "CREATE SPACE `test_comment_not_set` (partition_num = 100, replica_factor = 1, charset = utf8, collate = utf8_bin, vid_type = FIXED_STRING(8), atomic_edge = false) ON default" | + | Space | Create Space | + | "test_comment_not_set" | "CREATE SPACE `test_comment_not_set` (partition_num = 100, replica_factor = 1, charset = utf8, collate = utf8_bin, vid_type = FIXED_STRING(8), atomic_edge = false) ON default_zone" | When executing query: """ DESC SPACE test_comment_not_set; """ Then the result should be, in any order: - | ID | Name | Partition Number | Replica Factor | Charset | Collate | Vid Type | Atomic Edge | Group | Comment | - | /\d+/ | "test_comment_not_set" | 100 | 1 | "utf8" | "utf8_bin" | "FIXED_STRING(8)" | false | "default" | EMPTY | + | ID | Name | Partition Number | Replica Factor | Charset | Collate | Vid Type | Atomic Edge | Zones | Comment | + | /\d+/ | "test_comment_not_set" | 100 | 1 | "utf8" | "utf8_bin" | "FIXED_STRING(8)" | false | "default_zone" | EMPTY | When executing query: """ DROP SPACE test_comment_not_set; @@ -74,15 +74,15 @@ Feature: Schema Comment SHOW CREATE SPACE test_comment_empty; """ Then the result should be, in any order: - | Space | Create Space | - | "test_comment_empty" | "CREATE SPACE `test_comment_empty` (partition_num = 100, replica_factor = 1, charset = utf8, collate = utf8_bin, vid_type = FIXED_STRING(8), atomic_edge = false) ON default comment = ''" | + | Space | Create Space | + | "test_comment_empty" | "CREATE SPACE `test_comment_empty` (partition_num = 100, replica_factor = 1, charset = utf8, collate = utf8_bin, vid_type = FIXED_STRING(8), atomic_edge = false) ON default_zone comment = ''" | When executing query: """ DESC SPACE test_comment_empty; """ Then the result should be, in any order: - | ID | Name | Partition Number | Replica Factor | Charset | Collate | Vid Type | Atomic Edge | Group | Comment | - | /\d+/ | "test_comment_empty" | 100 | 1 | "utf8" | "utf8_bin" | "FIXED_STRING(8)" | false | "default" | "" | + | ID | Name | Partition Number | Replica Factor | Charset | Collate | Vid Type | Atomic Edge | Zones | Comment | + | /\d+/ | "test_comment_empty" | 100 | 1 | "utf8" | "utf8_bin" | "FIXED_STRING(8)" | false | "default_zone" | "" | When executing query: """ DROP SPACE test_comment_empty; From 9d1c1df384838ae83542e6314c07e5206fde4d27 Mon Sep 17 00:00:00 2001 From: "darion.yaphet" Date: Tue, 7 Dec 2021 17:48:38 +0800 Subject: [PATCH 4/5] fix critical27 comment --- src/graph/executor/admin/SpaceExecutor.cpp | 8 +-- src/meta/ActiveHostsMan.cpp | 5 +- .../zone/AddHostsIntoZoneProcessor.cpp | 4 +- .../processors/zone/AddHostsProcessor.cpp | 4 +- .../processors/zone/DropHostsProcessor.cpp | 68 ++++++++++++++---- src/meta/processors/zone/DropHostsProcessor.h | 5 +- .../processors/zone/RenameZoneProcessor.cpp | 2 + src/meta/test/ProcessorTest.cpp | 10 +++ src/parser/MaintainSentences.cpp | 22 +++--- src/parser/Sentence.h | 2 + src/parser/parser.yy | 18 ++--- src/parser/test/ParserTest.cpp | 70 ++++++++++++++++--- 12 files changed, 160 insertions(+), 58 deletions(-) diff --git a/src/graph/executor/admin/SpaceExecutor.cpp b/src/graph/executor/admin/SpaceExecutor.cpp index 801a97d2a7d..f2a452482fc 100644 --- a/src/graph/executor/admin/SpaceExecutor.cpp +++ b/src/graph/executor/admin/SpaceExecutor.cpp @@ -220,14 +220,12 @@ folly::Future ShowCreateSpaceExecutor::execute() { auto fmt = properties.comment_ref().has_value() ? "CREATE SPACE `%s` (partition_num = %d, replica_factor = %d, " "charset = %s, collate = %s, vid_type = %s, atomic_edge = %s" - ")%s" + ") ON %s" " comment = '%s'" : "CREATE SPACE `%s` (partition_num = %d, replica_factor = %d, " "charset = %s, collate = %s, vid_type = %s, atomic_edge = %s" - ")%s"; - auto zoneNames = !properties.get_zone_names().empty() - ? " ON " + folly::join(",", properties.get_zone_names()) - : ""; + ") ON %s"; + auto zoneNames = folly::join(",", properties.get_zone_names()); if (properties.comment_ref().has_value()) { row.values.emplace_back( folly::stringPrintf(fmt, diff --git a/src/meta/ActiveHostsMan.cpp b/src/meta/ActiveHostsMan.cpp index c870c5127b8..5a9b243de3c 100644 --- a/src/meta/ActiveHostsMan.cpp +++ b/src/meta/ActiveHostsMan.cpp @@ -102,11 +102,10 @@ ErrorOr> ActiveHostsMan::getActiv return retCode; } - std::vector machines; + std::unordered_set machines; while (machineIter->valid()) { auto machine = MetaKeyUtils::parseMachineKey(machineIter->key()); - LOG(INFO) << "Machine " << machine; - machines.emplace_back(std::move(machine)); + machines.emplace(std::move(machine)); machineIter->next(); } diff --git a/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp b/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp index 654cb7aabd0..274985fa8ec 100644 --- a/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp +++ b/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp @@ -11,8 +11,8 @@ namespace nebula { namespace meta { void AddHostsIntoZoneProcessor::process(const cpp2::AddHostsIntoZoneReq& req) { - folly::SharedMutex::ReadHolder zHolder(LockUtils::zoneLock()); - folly::SharedMutex::ReadHolder mHolder(LockUtils::machineLock()); + folly::SharedMutex::WriteHolder zHolder(LockUtils::zoneLock()); + folly::SharedMutex::WriteHolder mHolder(LockUtils::machineLock()); auto hosts = req.get_hosts(); // Confirm that there are no duplicates in the parameters. diff --git a/src/meta/processors/zone/AddHostsProcessor.cpp b/src/meta/processors/zone/AddHostsProcessor.cpp index 7672413a6e2..d765b917e29 100644 --- a/src/meta/processors/zone/AddHostsProcessor.cpp +++ b/src/meta/processors/zone/AddHostsProcessor.cpp @@ -11,8 +11,8 @@ namespace nebula { namespace meta { void AddHostsProcessor::process(const cpp2::AddHostsReq& req) { - folly::SharedMutex::ReadHolder zHolder(LockUtils::zoneLock()); - folly::SharedMutex::ReadHolder mHolder(LockUtils::machineLock()); + folly::SharedMutex::WriteHolder zHolder(LockUtils::zoneLock()); + folly::SharedMutex::WriteHolder mHolder(LockUtils::machineLock()); auto hosts = req.get_hosts(); // Confirm that there are no duplicates in the parameters. if (std::unique(hosts.begin(), hosts.end()) != hosts.end()) { diff --git a/src/meta/processors/zone/DropHostsProcessor.cpp b/src/meta/processors/zone/DropHostsProcessor.cpp index 12802889906..607e6f80042 100644 --- a/src/meta/processors/zone/DropHostsProcessor.cpp +++ b/src/meta/processors/zone/DropHostsProcessor.cpp @@ -9,6 +9,8 @@ namespace nebula { namespace meta { void DropHostsProcessor::process(const cpp2::DropHostsReq& req) { + folly::SharedMutex::WriteHolder zHolder(LockUtils::zoneLock()); + folly::SharedMutex::WriteHolder mHolder(LockUtils::machineLock()); auto hosts = req.get_hosts(); if (std::unique(hosts.begin(), hosts.end()) != hosts.end()) { LOG(ERROR) << "Hosts have duplicated element"; @@ -25,6 +27,7 @@ void DropHostsProcessor::process(const cpp2::DropHostsReq& req) { } std::vector data; + std::vector rewriteData; // Check that partition is not held on the host const auto& spacePrefix = MetaKeyUtils::spacePrefix(); auto spaceIterRet = doPrefix(spacePrefix); @@ -77,21 +80,49 @@ void DropHostsProcessor::process(const cpp2::DropHostsReq& req) { auto iter = nebula::value(iterRet).get(); while (iter->valid()) { + auto zoneKey = iter->key(); auto hs = MetaKeyUtils::parseZoneHosts(iter->val()); + // Delete all hosts in the zone if (std::all_of(hs.begin(), hs.end(), [&hosts](auto& h) { return std::find(hosts.begin(), hosts.end(), h) != hosts.end(); })) { - auto zoneName = MetaKeyUtils::parseZoneName(iter->key()); + auto zoneName = MetaKeyUtils::parseZoneName(zoneKey); LOG(INFO) << "Drop zone " << zoneName; - code = checkRelatedSpace(zoneName); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + auto result = checkRelatedSpaceAndCollect(zoneName); + if (!nebula::ok(result)) { + LOG(ERROR) << "Check related space failed"; + code = nebula::error(result); break; } - data.emplace_back(iter->key()); + + const auto& rewrites = nebula::value(result); + rewriteData.insert(rewriteData.end(), rewrites.begin(), rewrites.end()); + data.emplace_back(zoneKey); + } else { + // Delete some hosts in the zone + for (auto& h : hosts) { + auto it = std::find(hs.begin(), hs.end(), h); + if (it != hs.end()) { + hs.erase(it); + } + } + + auto zoneValue = MetaKeyUtils::zoneVal(hs); + LOG(INFO) << "Zone Value: " << zoneValue; + rewriteData.emplace_back(std::move(zoneKey), std::move(zoneValue)); + } + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + break; } iter->next(); } + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code); + onFinished(); + return; + } + // Detach the machine from cluster for (auto& host : hosts) { auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); @@ -111,10 +142,21 @@ void DropHostsProcessor::process(const cpp2::DropHostsReq& req) { } resp_.set_code(nebula::cpp2::ErrorCode::SUCCEEDED); - doMultiRemove(std::move(data)); + folly::Baton baton; + kvstore_->asyncMultiRemove(kDefaultSpaceId, + kDefaultPartId, + std::move(data), + [this, &baton](nebula::cpp2::ErrorCode result) { + this->handleErrorCode(result); + baton.post(); + }); + baton.wait(); + doSyncPutAndUpdate(std::move(rewriteData)); } -nebula::cpp2::ErrorCode DropHostsProcessor::checkRelatedSpace(const std::string& zoneName) { +ErrorOr> +DropHostsProcessor::checkRelatedSpaceAndCollect(const std::string& zoneName) { + std::vector data; const auto& prefix = MetaKeyUtils::spacePrefix(); auto ret = doPrefix(prefix); if (!nebula::ok(ret)) { @@ -137,19 +179,15 @@ nebula::cpp2::ErrorCode DropHostsProcessor::checkRelatedSpace(const std::string& } else { zones.erase(it); properties.set_zone_names(zones); - rewriteSpaceProperties(iter->key().data(), properties); + + auto spaceKey = iter->key().data(); + auto spaceVal = MetaKeyUtils::spaceVal(properties); + data.emplace_back(std::move(spaceKey), std::move(spaceVal)); } } iter->next(); } - return nebula::cpp2::ErrorCode::SUCCEEDED; -} - -void DropHostsProcessor::rewriteSpaceProperties(const std::string& spaceKey, - const meta::cpp2::SpaceDesc& properties) { - auto spaceVal = MetaKeyUtils::spaceVal(properties); - std::vector data = {{spaceKey, spaceVal}}; - doSyncPutAndUpdate(std::move(data)); + return data; } } // namespace meta diff --git a/src/meta/processors/zone/DropHostsProcessor.h b/src/meta/processors/zone/DropHostsProcessor.h index 41057636c02..8469eb79a45 100644 --- a/src/meta/processors/zone/DropHostsProcessor.h +++ b/src/meta/processors/zone/DropHostsProcessor.h @@ -22,9 +22,8 @@ class DropHostsProcessor : public BaseProcessor { private: explicit DropHostsProcessor(kvstore::KVStore* kvstore) : BaseProcessor(kvstore) {} - nebula::cpp2::ErrorCode checkRelatedSpace(const std::string& zoneName); - - void rewriteSpaceProperties(const std::string& spaceKey, const meta::cpp2::SpaceDesc& properties); + ErrorOr> checkRelatedSpaceAndCollect( + const std::string& zoneName); }; } // namespace meta diff --git a/src/meta/processors/zone/RenameZoneProcessor.cpp b/src/meta/processors/zone/RenameZoneProcessor.cpp index 4dcbf8fe5b5..56493689072 100644 --- a/src/meta/processors/zone/RenameZoneProcessor.cpp +++ b/src/meta/processors/zone/RenameZoneProcessor.cpp @@ -9,6 +9,8 @@ namespace nebula { namespace meta { void RenameZoneProcessor::process(const cpp2::RenameZoneReq& req) { + folly::SharedMutex::WriteHolder zHolder(LockUtils::zoneLock()); + auto originalZoneName = req.get_original_zone_name(); auto originalZoneKey = MetaKeyUtils::zoneKey(originalZoneName); auto originalZoneValueRet = doGet(std::move(originalZoneKey)); diff --git a/src/meta/test/ProcessorTest.cpp b/src/meta/test/ProcessorTest.cpp index 81cffadd621..417e279f9ea 100644 --- a/src/meta/test/ProcessorTest.cpp +++ b/src/meta/test/ProcessorTest.cpp @@ -3303,11 +3303,17 @@ TEST(ProcessorTest, DropHostsTest) { ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); ASSERT_EQ(6, resp.get_zones().size()); ASSERT_EQ("default_zone_127.0.0.1_8987", resp.get_zones()[0].get_zone_name()); + ASSERT_EQ(1, resp.get_zones()[0].get_nodes().size()); ASSERT_EQ("default_zone_127.0.0.1_8988", resp.get_zones()[1].get_zone_name()); + ASSERT_EQ(1, resp.get_zones()[1].get_nodes().size()); ASSERT_EQ("default_zone_127.0.0.1_8989", resp.get_zones()[2].get_zone_name()); + ASSERT_EQ(1, resp.get_zones()[2].get_nodes().size()); ASSERT_EQ("zone_0", resp.get_zones()[3].get_zone_name()); + ASSERT_EQ(2, resp.get_zones()[3].get_nodes().size()); ASSERT_EQ("zone_1", resp.get_zones()[4].get_zone_name()); + ASSERT_EQ(1, resp.get_zones()[4].get_nodes().size()); ASSERT_EQ("zone_2", resp.get_zones()[5].get_zone_name()); + ASSERT_EQ(1, resp.get_zones()[5].get_nodes().size()); } { // Create Space on cluster, the replica number same with the zone size @@ -3466,9 +3472,13 @@ TEST(ProcessorTest, DropHostsTest) { ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); ASSERT_EQ(4, resp.get_zones().size()); ASSERT_EQ("default_zone_127.0.0.1_8988", resp.get_zones()[0].get_zone_name()); + ASSERT_EQ(1, resp.get_zones()[0].get_nodes().size()); ASSERT_EQ("default_zone_127.0.0.1_8989", resp.get_zones()[1].get_zone_name()); + ASSERT_EQ(1, resp.get_zones()[1].get_nodes().size()); ASSERT_EQ("zone_0", resp.get_zones()[2].get_zone_name()); + ASSERT_EQ(1, resp.get_zones()[2].get_nodes().size()); ASSERT_EQ("zone_2", resp.get_zones()[3].get_zone_name()); + ASSERT_EQ(1, resp.get_zones()[3].get_nodes().size()); } } diff --git a/src/parser/MaintainSentences.cpp b/src/parser/MaintainSentences.cpp index 78cf358e57a..b9ced49661c 100644 --- a/src/parser/MaintainSentences.cpp +++ b/src/parser/MaintainSentences.cpp @@ -379,37 +379,40 @@ std::string MergeZoneSentence::toString() const { buf.reserve(128); buf += "MERGE ZONE "; buf += zoneNames_->toString(); - buf += " INTO "; + buf += " INTO \""; buf += *zoneName_; + buf += "\""; return buf; } std::string DropZoneSentence::toString() const { - return folly::stringPrintf("DROP ZONE %s", zoneName_.get()->c_str()); + return folly::stringPrintf("DROP ZONE \"%s\"", zoneName_.get()->c_str()); } std::string SplitZoneSentence::toString() const { std::string buf; buf.reserve(128); - buf += "SPLIT ZONE "; + buf += "SPLIT ZONE \""; buf += *zoneName_; - buf += " INTO "; + buf += "\" INTO \""; buf += zoneNames_->toString(); + buf += "\""; return buf; } std::string RenameZoneSentence::toString() const { std::string buf; buf.reserve(128); - buf += "RENAME ZONE "; + buf += "RENAME ZONE \""; buf += *originalZoneName_; - buf += " TO "; + buf += "\" TO \""; buf += *zoneName_; + buf += "\""; return buf; } std::string DescribeZoneSentence::toString() const { - return folly::stringPrintf("DESCRIBE ZONE %s", zoneName_.get()->c_str()); + return folly::stringPrintf("DESCRIBE ZONE \"%s\"", zoneName_.get()->c_str()); } std::string ListZonesSentence::toString() const { return folly::stringPrintf("SHOW ZONES"); } @@ -420,11 +423,12 @@ std::string AddHostsIntoZoneSentence::toString() const { buf += "ADD HOSTS "; buf += address_->toString(); if (isNew_) { - buf += " INTO NEW ZONE "; + buf += " INTO NEW ZONE \""; } else { - buf += " INTO ZONE "; + buf += " INTO ZONE \""; } buf += *zoneName_; + buf += "\""; return buf; } diff --git a/src/parser/Sentence.h b/src/parser/Sentence.h index 9af5ebe410d..4e27150d368 100644 --- a/src/parser/Sentence.h +++ b/src/parser/Sentence.h @@ -205,7 +205,9 @@ class ZoneNameList final { std::string toString() const { std::string buf; for (const auto &zone : zones_) { + buf += "\""; buf += *zone; + buf += "\""; buf += ","; } if (!zones_.empty()) { diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 5206c6d2d67..93c38013add 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -2686,10 +2686,10 @@ add_hosts_sentence : KW_ADD KW_HOSTS host_list { $$ = new AddHostsSentence($3); } - | KW_ADD KW_HOSTS host_list KW_INTO KW_ZONE name_label { + | KW_ADD KW_HOSTS host_list KW_INTO KW_ZONE STRING { $$ = new AddHostsIntoZoneSentence($3, $6, false); } - | KW_ADD KW_HOSTS host_list KW_INTO KW_NEW KW_ZONE name_label { + | KW_ADD KW_HOSTS host_list KW_INTO KW_NEW KW_ZONE STRING { $$ = new AddHostsIntoZoneSentence($3, $7, true); } ; @@ -2702,13 +2702,13 @@ drop_hosts_sentence merge_zone_sentence - : KW_MERGE KW_ZONE zone_name_list KW_INTO name_label { + : KW_MERGE KW_ZONE zone_name_list KW_INTO STRING { $$ = new MergeZoneSentence($3, $5); } ; drop_zone_sentence - : KW_DROP KW_ZONE name_label { + : KW_DROP KW_ZONE STRING { $$ = new DropZoneSentence($3); } ; @@ -2720,16 +2720,16 @@ drop_zone_sentence // ; rename_zone_sentence - : KW_RENAME KW_ZONE name_label KW_TO name_label { + : KW_RENAME KW_ZONE STRING KW_TO STRING { $$ = new RenameZoneSentence($3, $5); } ; desc_zone_sentence - : KW_DESCRIBE KW_ZONE name_label { + : KW_DESCRIBE KW_ZONE STRING { $$ = new DescribeZoneSentence($3); } - | KW_DESC KW_ZONE name_label { + | KW_DESC KW_ZONE STRING { $$ = new DescribeZoneSentence($3); } ; @@ -3307,11 +3307,11 @@ show_config_item ; zone_name_list - : name_label { + : STRING { $$ = new ZoneNameList(); $$->addZone($1); } - | zone_name_list COMMA name_label { + | zone_name_list COMMA STRING { $$ = $1; $$->addZone($3); } diff --git a/src/parser/test/ParserTest.cpp b/src/parser/test/ParserTest.cpp index 1fa8d794979..022ab938d56 100644 --- a/src/parser/test/ParserTest.cpp +++ b/src/parser/test/ParserTest.cpp @@ -227,12 +227,24 @@ TEST_F(ParserTest, SpaceOperation) { { std::string query = "CREATE SPACE default_space(partition_num=9, replica_factor=3) " - "ON group_0"; + "ON \"zone_0\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "CREATE SPACE default_space ON group_0"; + std::string query = + "CREATE SPACE default_space(partition_num=9, replica_factor=3) " + "ON \"zone_0\",\"zone_1\",\"zone_2\""; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "CREATE SPACE default_space ON \"zone_0\""; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "CREATE SPACE default_space ON \"zone_0\",\"zone_1\",\"zone_2\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } @@ -2768,42 +2780,80 @@ TEST_F(ParserTest, Zone) { ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "ADD HOSTS 127.0.0.1:8989 INTO ZONE zone_0"; + std::string query = "ADD HOSTS 127.0.0.1:8989 INTO ZONE \"zone_0\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "ADD HOSTS 127.0.0.1:8988,127.0.0.1:8989 INTO ZONE zone_0"; + std::string query = "ADD HOSTS 127.0.0.1:8989 INTO ZONE \"default_zone_127.0.0.1_8988\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "DESC ZONE zone_0"; + std::string query = "ADD HOSTS 127.0.0.1:8988,127.0.0.1:8989 INTO ZONE \"zone_0\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "DESCRIBE ZONE zone_0"; + std::string query = + "ADD HOSTS 127.0.0.1:8988,127.0.0.1:8989 INTO ZONE" + " \"default_zone_127.0.0.1_8988\""; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "DESC ZONE \"zone_0\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "DROP ZONE zone_0"; + std::string query = "DESCRIBE ZONE \"zone_0\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "MERGE ZONE zone_1,zone_2 INTO zone"; + std::string query = "DROP ZONE \"zone_0\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "MERGE ZONE zone_1,zone_2 INTO zone_1"; + std::string query = "MERGE ZONE \"zone_1\",\"zone_2\" INTO \"zone\""; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "MERGE ZONE \"zone_1\",\"zone_2\" INTO \"zone_1\""; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = + "MERGE ZONE \"default_zone_127.0.0.1_8988\",\"default_zone_127.0.0.1_8989\"" + "INTO \"zone_1\""; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = + "MERGE ZONE \"default_zone_127.0.0.1_8988\",\"default_zone_127.0.0.1_8989\"" + "INTO \"default_zone_127.0.0.1_8989\""; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + + { + std::string query = "RENAME ZONE \"default_zone_127.0.0.1_8989\" TO \"new_name\""; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + + { + std::string query = "RENAME ZONE \"old_name\" TO \"new_name\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } { - std::string query = "RENAME ZONE old_name TO new_name"; + std::string query = "RENAME ZONE \"default_zone_127.0.0.1_8989\" TO \"new_name\""; auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } From 283e686c6196b6be353ffa9c60f4a3ca341bd1a2 Mon Sep 17 00:00:00 2001 From: "darion.yaphet" Date: Tue, 14 Dec 2021 18:07:51 +0800 Subject: [PATCH 5/5] fix python test case --- tests/common/nebula_service.py | 2 +- tests/tck/features/admin/Hosts.feature | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/common/nebula_service.py b/tests/common/nebula_service.py index 502e79a38e2..13dcad97076 100644 --- a/tests/common/nebula_service.py +++ b/tests/common/nebula_service.py @@ -366,7 +366,7 @@ def start(self): # assert client_pool.init([("127.0.0.1", int(self.graphd_port))], config) assert client_pool.init([("127.0.0.1", self.graphd_processes[0].tcp_port)], config) - cmd = "ADD HOSTS 127.0.0.1:" + str(self.storaged_processes[0].tcp_port) + " INTO NEW ZONE default_zone" + cmd = "ADD HOSTS 127.0.0.1:" + str(self.storaged_processes[0].tcp_port) + " INTO NEW ZONE \"default_zone\"" print(cmd) # get session from the pool diff --git a/tests/tck/features/admin/Hosts.feature b/tests/tck/features/admin/Hosts.feature index 2a70092f1e9..0d1b629cb31 100644 --- a/tests/tck/features/admin/Hosts.feature +++ b/tests/tck/features/admin/Hosts.feature @@ -46,12 +46,12 @@ Feature: Admin hosts Then a SemanticError should be raised at runtime: space vid_type must be specified explicitly When executing query: """ - CREATE SPACE space_without_vid_type(partition_num=9, replica_factor=3) on group_0; + CREATE SPACE space_without_vid_type(partition_num=9, replica_factor=3) on "default_zone"; """ Then a SemanticError should be raised at runtime: space vid_type must be specified explicitly When executing query: """ - CREATE SPACE space_without_vid_type on group_0; + CREATE SPACE space_without_vid_type on "default_zone"; """ Then a SemanticError should be raised at runtime: space vid_type must be specified explicitly When executing query: