From a561f39592f8a8d8870c024f0dfddd7a0679a3b0 Mon Sep 17 00:00:00 2001 From: Simon Liu <331435+monadbobo@users.noreply.github.com> Date: Thu, 12 Sep 2019 09:28:12 +0800 Subject: [PATCH 1/3] Issue #931 the "meta_server_addrs" in the configuration support the host name. Modify the toHostAddr interface to support host resolved (gethostbyname). --- src/common/network/NetworkUtils.cpp | 31 ++++++++++++---- src/common/network/NetworkUtils.h | 9 ++--- src/common/network/test/NetworkUtilsTest.cpp | 39 +++++++++++++++++++- src/daemons/MetaDaemon.cpp | 4 +- src/daemons/StorageDaemon.cpp | 2 +- src/graph/test/TestEnv.cpp | 2 +- src/storage/test/StorageClientTest.cpp | 2 +- 7 files changed, 71 insertions(+), 18 deletions(-) diff --git a/src/common/network/NetworkUtils.cpp b/src/common/network/NetworkUtils.cpp index 3a1e19899af..326ad55aa96 100644 --- a/src/common/network/NetworkUtils.cpp +++ b/src/common/network/NetworkUtils.cpp @@ -215,15 +215,32 @@ std::string NetworkUtils::intToIPv4(IPv4 ip) { return buf; } -StatusOr NetworkUtils::toHostAddr(folly::StringPiece ip, int32_t port) { +StatusOr> NetworkUtils::toHostAddr(folly::StringPiece ip, int32_t port) { + std::vector addrs; IPv4 ipV4; - if (!ipv4ToInt(ip.toString(), ipV4)) { - return Status::Error("Bad ip format:%s", ip.start()); + std::string str = ip.toString(); + if (ipv4ToInt(str, ipV4)) { + addrs.emplace_back(std::move(ipV4), port); + return addrs; } - return std::make_pair(ipV4, port); + + struct hostent* h; + h = gethostbyname(str.c_str()); + if (h == NULL || h->h_addr_list[0] == NULL) { + return Status::Error("host not found:%s", ip.start()); + } + + for (auto i = 0; h->h_addr_list[i] != NULL; i++) { + // We need to match the integer byte order generated by ipv4ToInt, so we need to convert + // here. + addrs.emplace_back(htonl(std::move(*reinterpret_cast(h->h_addr_list[i]))), + port); + } + + return addrs; } -StatusOr NetworkUtils::toHostAddr(folly::StringPiece ipPort) { +StatusOr> NetworkUtils::toHostAddr(folly::StringPiece ipPort) { auto pos = ipPort.find(':'); if (pos == folly::StringPiece::npos) { return Status::Error("Bad peer format: %s", ipPort.start()); @@ -249,7 +266,8 @@ StatusOr> NetworkUtils::toHosts(const std::string& peersSt if (!hostAddr.ok()) { return hostAddr.status(); } - hosts.emplace_back(hostAddr.value()); + hosts.insert(hosts.end(), std::make_move_iterator(hostAddr.value().begin()), + std::make_move_iterator(hostAddr.value().end())); } return hosts; } @@ -281,4 +299,3 @@ StatusOr NetworkUtils::getLocalIP(std::string defaultIP) { } // namespace network } // namespace nebula - diff --git a/src/common/network/NetworkUtils.h b/src/common/network/NetworkUtils.h index 95229fa1478..3d836548ffe 100644 --- a/src/common/network/NetworkUtils.h +++ b/src/common/network/NetworkUtils.h @@ -33,10 +33,10 @@ class NetworkUtils final { // Get a dynamic port that is not in use static uint16_t getAvailablePort(); - // Convert the given IP (must be in the form of "xx.xx.xx.xx") and Port to a HostAddr - static StatusOr toHostAddr(folly::StringPiece ip, int32_t port); - // Convert the given IP/Port (must be in the form of "xx.xx.xx.xx:pp") to a HostAddr - static StatusOr toHostAddr(folly::StringPiece ipPort); + // Convert the given IP/HOST and Port to a HostAddr + static StatusOr> toHostAddr(folly::StringPiece ip, int32_t port); + // Convert the given (IP/HOST):Port to a HostAddr + static StatusOr> toHostAddr(folly::StringPiece ipPort); // Retrieve the string-form IP from the given HostAddr static std::string ipFromHostAddr(const HostAddr& host); // Retrieve the port number from the given HostAddr @@ -67,4 +67,3 @@ class NetworkUtils final { } // namespace nebula #endif // COMMON_NETWORK_NETWORKUTILS_H_ - diff --git a/src/common/network/test/NetworkUtilsTest.cpp b/src/common/network/test/NetworkUtilsTest.cpp index 3ef407e54c0..bf1b894871c 100644 --- a/src/common/network/test/NetworkUtilsTest.cpp +++ b/src/common/network/test/NetworkUtilsTest.cpp @@ -87,6 +87,44 @@ TEST(NetworkUtils, getAvailablePort) { ASSERT_GT(port, 0); } +TEST(NetworkUtils, toHostAddr) { + auto s = NetworkUtils::toHostAddr("localhost:1200"); + ASSERT_TRUE(s.ok()); + auto addr = s.value(); + IPv4 ip; + ASSERT_TRUE(NetworkUtils::ipv4ToInt("127.0.0.1", ip)); + ASSERT_EQ(addr[0].first, ip); + ASSERT_EQ(addr[0].second, 1200); + + s = NetworkUtils::toHostAddr("8.8.8.8:1300"); + ASSERT_TRUE(s.ok()); + addr = s.value(); + + ASSERT_TRUE(NetworkUtils::ipv4ToInt("8.8.8.8", ip)); + ASSERT_EQ(addr[0].first, ip); + ASSERT_EQ(addr[0].second, 1300); + + s = NetworkUtils::toHostAddr("a.b.c.d:a23"); + ASSERT_FALSE(s.ok()); +} + +TEST(NetworkUtils, toHosts) { + auto s = NetworkUtils::toHosts("localhost:1200, 127.0.0.1:1200"); + ASSERT_TRUE(s.ok()); + auto addr = s.value(); + + IPv4 ip; + ASSERT_TRUE(NetworkUtils::ipv4ToInt("127.0.0.1", ip)); + ASSERT_EQ(addr[0].first, ip); + ASSERT_EQ(addr[0].second, 1200); + + ASSERT_EQ(addr[1].first, ip); + ASSERT_EQ(addr[1].second, 1200); + + s = NetworkUtils::toHostAddr("1.1.2.3:123, a.b.c.d:a23"); + ASSERT_FALSE(s.ok()); +} + } // namespace network } // namespace nebula @@ -98,4 +136,3 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } - diff --git a/src/daemons/MetaDaemon.cpp b/src/daemons/MetaDaemon.cpp index fcc7e89ffc0..c0c62b48412 100644 --- a/src/daemons/MetaDaemon.cpp +++ b/src/daemons/MetaDaemon.cpp @@ -203,14 +203,14 @@ int main(int argc, char *argv[]) { LOG(ERROR) << "Bad local host addr, status:" << hostAddrRet.status(); return EXIT_FAILURE; } - auto& localhost = hostAddrRet.value(); + auto& localhost = hostAddrRet.value()[0]; auto peersRet = nebula::network::NetworkUtils::toHosts(FLAGS_meta_server_addrs); if (!peersRet.ok()) { LOG(ERROR) << "Can't get peers address, status:" << peersRet.status(); return EXIT_FAILURE; } - auto kvstore = initKV(peersRet.value(), hostAddrRet.value()); + auto kvstore = initKV(peersRet.value(), localhost); if (kvstore == nullptr) { LOG(ERROR) << "Init kv failed!"; return EXIT_FAILURE; diff --git a/src/daemons/StorageDaemon.cpp b/src/daemons/StorageDaemon.cpp index e6f595cce99..f13265984af 100644 --- a/src/daemons/StorageDaemon.cpp +++ b/src/daemons/StorageDaemon.cpp @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } - gStorageServer = std::make_unique(hostRet.value(), + gStorageServer = std::make_unique(hostRet.value()[0], metaAddrsRet.value(), paths); if (!gStorageServer->start()) { diff --git a/src/graph/test/TestEnv.cpp b/src/graph/test/TestEnv.cpp index b4906e45826..9162dd171b0 100644 --- a/src/graph/test/TestEnv.cpp +++ b/src/graph/test/TestEnv.cpp @@ -46,7 +46,7 @@ void TestEnv::SetUp() { if (!hostRet.ok()) { LOG(ERROR) << "Bad local host addr, status:" << hostRet.status(); } - auto& localhost = hostRet.value(); + auto& localhost = hostRet.value()[0]; mClient_ = std::make_unique(threadPool, std::move(addrsRet.value()), diff --git a/src/storage/test/StorageClientTest.cpp b/src/storage/test/StorageClientTest.cpp index 727e930756c..7c5a3e91eaa 100644 --- a/src/storage/test/StorageClientTest.cpp +++ b/src/storage/test/StorageClientTest.cpp @@ -48,7 +48,7 @@ TEST(StorageClientTest, VerticesInterfacesTest) { auto& addrs = addrsRet.value(); uint32_t localDataPort = network::NetworkUtils::getAvailablePort(); auto hostRet = nebula::network::NetworkUtils::toHostAddr("127.0.0.1", localDataPort); - auto& localHost = hostRet.value(); + auto& localHost = hostRet.value()[0]; auto mClient = std::make_unique(threadPool, std::move(addrs), localHost, From eadd36812181790a2ec82ef5f7237fac6ee22429 Mon Sep 17 00:00:00 2001 From: Simon Liu <331435+monadbobo@users.noreply.github.com> Date: Mon, 16 Sep 2019 16:54:50 +0800 Subject: [PATCH 2/3] replace the gethostbyname with getaddrinfo. --- src/common/network/NetworkUtils.cpp | 33 +++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/common/network/NetworkUtils.cpp b/src/common/network/NetworkUtils.cpp index 326ad55aa96..cd9855899fa 100644 --- a/src/common/network/NetworkUtils.cpp +++ b/src/common/network/NetworkUtils.cpp @@ -224,17 +224,38 @@ StatusOr> NetworkUtils::toHostAddr(folly::StringPiece ip, return addrs; } - struct hostent* h; - h = gethostbyname(str.c_str()); - if (h == NULL || h->h_addr_list[0] == NULL) { + struct addrinfo hints, *res, *rp; + ::memset(&hints, 0, sizeof(struct addrinfo)); + + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_ADDRCONFIG; + + if (getaddrinfo(str.c_str(), nullptr, &hints, &res) != 0) { return Status::Error("host not found:%s", ip.start()); } - for (auto i = 0; h->h_addr_list[i] != NULL; i++) { + for (rp = res; rp != nullptr; rp = rp->ai_next) { + switch (rp->ai_family) { + case AF_INET: + break; + case AF_INET6: + VLOG(1) << "Currently does not support Ipv6 address"; + continue; + default: + continue; + } + + auto address = ((struct sockaddr_in*)rp->ai_addr)->sin_addr.s_addr; // We need to match the integer byte order generated by ipv4ToInt, so we need to convert // here. - addrs.emplace_back(htonl(std::move(*reinterpret_cast(h->h_addr_list[i]))), - port); + addrs.emplace_back(htonl(std::move(address)), port); + } + + freeaddrinfo(res); + + if (addrs.empty()) { + return Status::Error("host not found: %s", str.c_str()); } return addrs; From 5f8d7bde911d6603f88f9d11c398cc0012bda62a Mon Sep 17 00:00:00 2001 From: Simon Liu <331435+monadbobo@users.noreply.github.com> Date: Tue, 17 Sep 2019 23:22:23 +0800 Subject: [PATCH 3/3] Refactoring part of the code. --- src/common/network/NetworkUtils.cpp | 64 +++++++++++--------- src/common/network/NetworkUtils.h | 5 +- src/common/network/test/NetworkUtilsTest.cpp | 18 +++--- src/daemons/MetaDaemon.cpp | 2 +- src/daemons/StorageDaemon.cpp | 2 +- src/graph/test/TestEnv.cpp | 2 +- src/storage/test/StorageClientTest.cpp | 2 +- 7 files changed, 51 insertions(+), 44 deletions(-) diff --git a/src/common/network/NetworkUtils.cpp b/src/common/network/NetworkUtils.cpp index cd9855899fa..0f9c11a5ff3 100644 --- a/src/common/network/NetworkUtils.cpp +++ b/src/common/network/NetworkUtils.cpp @@ -215,15 +215,8 @@ std::string NetworkUtils::intToIPv4(IPv4 ip) { return buf; } -StatusOr> NetworkUtils::toHostAddr(folly::StringPiece ip, int32_t port) { +StatusOr> NetworkUtils::resolveHost(const std::string &ip, int32_t port) { std::vector addrs; - IPv4 ipV4; - std::string str = ip.toString(); - if (ipv4ToInt(str, ipV4)) { - addrs.emplace_back(std::move(ipV4), port); - return addrs; - } - struct addrinfo hints, *res, *rp; ::memset(&hints, 0, sizeof(struct addrinfo)); @@ -231,8 +224,8 @@ StatusOr> NetworkUtils::toHostAddr(folly::StringPiece ip, hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_ADDRCONFIG; - if (getaddrinfo(str.c_str(), nullptr, &hints, &res) != 0) { - return Status::Error("host not found:%s", ip.start()); + if (getaddrinfo(ip.c_str(), nullptr, &hints, &res) != 0) { + return Status::Error("host not found:%s", ip.c_str()); } for (rp = res; rp != nullptr; rp = rp->ai_next) { @@ -255,26 +248,19 @@ StatusOr> NetworkUtils::toHostAddr(folly::StringPiece ip, freeaddrinfo(res); if (addrs.empty()) { - return Status::Error("host not found: %s", str.c_str()); + return Status::Error("host not found: %s", ip.c_str()); } return addrs; } -StatusOr> NetworkUtils::toHostAddr(folly::StringPiece ipPort) { - auto pos = ipPort.find(':'); - if (pos == folly::StringPiece::npos) { - return Status::Error("Bad peer format: %s", ipPort.start()); - } - int32_t port; - try { - port = folly::to(ipPort.subpiece(pos + 1)); - } catch (const std::exception& ex) { - return Status::Error("Bad port number, error: %s", ex.what()); +StatusOr NetworkUtils::toHostAddr(const std::string &ip, int32_t port) { + IPv4 ipV4; + if (!ipv4ToInt(ip, ipV4)) { + return Status::Error("Bad ip format:%s", ip.c_str()); } - - return toHostAddr(ipPort.subpiece(0, pos), port); + return std::make_pair(ipV4, port); } StatusOr> NetworkUtils::toHosts(const std::string& peersStr) { @@ -283,12 +269,34 @@ StatusOr> NetworkUtils::toHosts(const std::string& peersSt folly::split(",", peersStr, peers, true); hosts.reserve(peers.size()); for (auto& peerStr : peers) { - auto hostAddr = network::NetworkUtils::toHostAddr(folly::trimWhitespace(peerStr)); - if (!hostAddr.ok()) { - return hostAddr.status(); + auto ipPort = folly::trimWhitespace(peerStr); + auto pos = ipPort.find(':'); + if (pos == folly::StringPiece::npos) { + return Status::Error("Bad peer format: %s", ipPort.start()); } - hosts.insert(hosts.end(), std::make_move_iterator(hostAddr.value().begin()), - std::make_move_iterator(hostAddr.value().end())); + + int32_t port; + try { + port = folly::to(ipPort.subpiece(pos + 1)); + } catch (const std::exception& ex) { + return Status::Error("Bad port number, error: %s", ex.what()); + } + + auto ipAddr = ipPort.subpiece(0, pos).toString(); + auto hostAddr = toHostAddr(ipAddr, port); + if (hostAddr.ok()) { + hosts.emplace_back(hostAddr.value()); + continue; + } + + auto resolveAddr = resolveHost(ipAddr, port); + if (resolveAddr.ok()) { + hosts.insert(hosts.end(), std::make_move_iterator(resolveAddr.value().begin()), + std::make_move_iterator(resolveAddr.value().end())); + continue; + } + + return resolveAddr.status(); } return hosts; } diff --git a/src/common/network/NetworkUtils.h b/src/common/network/NetworkUtils.h index 3d836548ffe..39c7e6097af 100644 --- a/src/common/network/NetworkUtils.h +++ b/src/common/network/NetworkUtils.h @@ -33,10 +33,9 @@ class NetworkUtils final { // Get a dynamic port that is not in use static uint16_t getAvailablePort(); + static StatusOr> resolveHost(const std::string &ip, int32_t port); // Convert the given IP/HOST and Port to a HostAddr - static StatusOr> toHostAddr(folly::StringPiece ip, int32_t port); - // Convert the given (IP/HOST):Port to a HostAddr - static StatusOr> toHostAddr(folly::StringPiece ipPort); + static StatusOr toHostAddr(const std::string &ip, int32_t port); // Retrieve the string-form IP from the given HostAddr static std::string ipFromHostAddr(const HostAddr& host); // Retrieve the port number from the given HostAddr diff --git a/src/common/network/test/NetworkUtilsTest.cpp b/src/common/network/test/NetworkUtilsTest.cpp index bf1b894871c..5be65133ce5 100644 --- a/src/common/network/test/NetworkUtilsTest.cpp +++ b/src/common/network/test/NetworkUtilsTest.cpp @@ -88,7 +88,7 @@ TEST(NetworkUtils, getAvailablePort) { } TEST(NetworkUtils, toHostAddr) { - auto s = NetworkUtils::toHostAddr("localhost:1200"); + auto s = NetworkUtils::resolveHost("localhost", 1200); ASSERT_TRUE(s.ok()); auto addr = s.value(); IPv4 ip; @@ -96,16 +96,16 @@ TEST(NetworkUtils, toHostAddr) { ASSERT_EQ(addr[0].first, ip); ASSERT_EQ(addr[0].second, 1200); - s = NetworkUtils::toHostAddr("8.8.8.8:1300"); - ASSERT_TRUE(s.ok()); - addr = s.value(); + auto s2 = NetworkUtils::toHostAddr("8.8.8.8", 1300); + ASSERT_TRUE(s2.ok()); + auto addr2 = s2.value(); ASSERT_TRUE(NetworkUtils::ipv4ToInt("8.8.8.8", ip)); - ASSERT_EQ(addr[0].first, ip); - ASSERT_EQ(addr[0].second, 1300); + ASSERT_EQ(addr2.first, ip); + ASSERT_EQ(addr2.second, 1300); - s = NetworkUtils::toHostAddr("a.b.c.d:a23"); - ASSERT_FALSE(s.ok()); + s2 = NetworkUtils::toHostAddr("a.b.c.d:a23", 1200); + ASSERT_FALSE(s2.ok()); } TEST(NetworkUtils, toHosts) { @@ -121,7 +121,7 @@ TEST(NetworkUtils, toHosts) { ASSERT_EQ(addr[1].first, ip); ASSERT_EQ(addr[1].second, 1200); - s = NetworkUtils::toHostAddr("1.1.2.3:123, a.b.c.d:a23"); + s = NetworkUtils::toHosts("1.1.2.3:123, a.b.c.d:a23"); ASSERT_FALSE(s.ok()); } diff --git a/src/daemons/MetaDaemon.cpp b/src/daemons/MetaDaemon.cpp index c0c62b48412..a77017fe7a9 100644 --- a/src/daemons/MetaDaemon.cpp +++ b/src/daemons/MetaDaemon.cpp @@ -203,7 +203,7 @@ int main(int argc, char *argv[]) { LOG(ERROR) << "Bad local host addr, status:" << hostAddrRet.status(); return EXIT_FAILURE; } - auto& localhost = hostAddrRet.value()[0]; + auto& localhost = hostAddrRet.value(); auto peersRet = nebula::network::NetworkUtils::toHosts(FLAGS_meta_server_addrs); if (!peersRet.ok()) { LOG(ERROR) << "Can't get peers address, status:" << peersRet.status(); diff --git a/src/daemons/StorageDaemon.cpp b/src/daemons/StorageDaemon.cpp index f13265984af..e6f595cce99 100644 --- a/src/daemons/StorageDaemon.cpp +++ b/src/daemons/StorageDaemon.cpp @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } - gStorageServer = std::make_unique(hostRet.value()[0], + gStorageServer = std::make_unique(hostRet.value(), metaAddrsRet.value(), paths); if (!gStorageServer->start()) { diff --git a/src/graph/test/TestEnv.cpp b/src/graph/test/TestEnv.cpp index 9162dd171b0..b4906e45826 100644 --- a/src/graph/test/TestEnv.cpp +++ b/src/graph/test/TestEnv.cpp @@ -46,7 +46,7 @@ void TestEnv::SetUp() { if (!hostRet.ok()) { LOG(ERROR) << "Bad local host addr, status:" << hostRet.status(); } - auto& localhost = hostRet.value()[0]; + auto& localhost = hostRet.value(); mClient_ = std::make_unique(threadPool, std::move(addrsRet.value()), diff --git a/src/storage/test/StorageClientTest.cpp b/src/storage/test/StorageClientTest.cpp index 7c5a3e91eaa..727e930756c 100644 --- a/src/storage/test/StorageClientTest.cpp +++ b/src/storage/test/StorageClientTest.cpp @@ -48,7 +48,7 @@ TEST(StorageClientTest, VerticesInterfacesTest) { auto& addrs = addrsRet.value(); uint32_t localDataPort = network::NetworkUtils::getAvailablePort(); auto hostRet = nebula::network::NetworkUtils::toHostAddr("127.0.0.1", localDataPort); - auto& localHost = hostRet.value()[0]; + auto& localHost = hostRet.value(); auto mClient = std::make_unique(threadPool, std::move(addrs), localHost,