From 9b7e49828b7e34e47e4b0145e375eb7ac9d70e09 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 1 Jun 2018 13:31:38 -0500 Subject: [PATCH] common/pick_addresses: new addrvec-based pick_addresses() Flags for everything we would want. The intent is to return an addrvec of addrs that we can directly bind to. Signed-off-by: Sage Weil --- src/common/pick_address.cc | 178 ++++++++++++++++++++++++++++++++- src/common/pick_address.h | 5 + src/test/test_ipaddr.cc | 197 +++++++++++++++++++++++++++++++++++++ 3 files changed, 378 insertions(+), 2 deletions(-) diff --git a/src/common/pick_address.cc b/src/common/pick_address.cc index f25ff65af7d12..6cc8ca8b458b0 100644 --- a/src/common/pick_address.cc +++ b/src/common/pick_address.cc @@ -215,6 +215,179 @@ void pick_addresses(CephContext *cct, int needs) freeifaddrs(ifa); } +static int fill_in_one_address( + CephContext *cct, + const struct ifaddrs *ifa, + unsigned ipv, + const string networks, + const string interfaces, + entity_addrvec_t *addrs) +{ + const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv, networks, + interfaces); + if (!found) { + return -1; + } + + char buf[INET6_ADDRSTRLEN]; + int err; + + err = getnameinfo(found, + (found->sa_family == AF_INET) + ? sizeof(struct sockaddr_in) + : sizeof(struct sockaddr_in6), + + buf, sizeof(buf), + nullptr, 0, + NI_NUMERICHOST); + if (err != 0) { + return -1; + } + + entity_addr_t addr; + const char *end = 0; + bool r = addr.parse(buf, &end); + if (!r) { + return -1; + } + addrs->v.push_back(addr); + return 0; +} + +int pick_addresses( + CephContext *cct, + unsigned flags, + struct ifaddrs *ifa, + entity_addrvec_t *addrs) +{ + addrs->v.clear(); + + unsigned addrt = (flags & (CEPH_PICK_ADDRESS_PUBLIC | + CEPH_PICK_ADDRESS_CLUSTER)); + if (addrt == 0 || + addrt == (CEPH_PICK_ADDRESS_PUBLIC | + CEPH_PICK_ADDRESS_CLUSTER)) { + return -EINVAL; + } + unsigned msgrv = flags & (CEPH_PICK_ADDRESS_MSGR1 | + CEPH_PICK_ADDRESS_MSGR2); + if (msgrv == 0) { + return -EINVAL; + } + unsigned ipv = flags & (CEPH_PICK_ADDRESS_IPV4 | + CEPH_PICK_ADDRESS_IPV6); + if (ipv == 0) { + return -EINVAL; + } + + entity_addr_t addr; + string networks; + string interfaces; + if (addrt & CEPH_PICK_ADDRESS_PUBLIC) { + addr = cct->_conf->get_val("public_addr"); + networks = cct->_conf->get_val("public_network"); + interfaces = + cct->_conf->get_val("public_network_interface"); + } else { + addr = cct->_conf->get_val("cluster_addr"); + networks = cct->_conf->get_val("cluster_network"); + interfaces = + cct->_conf->get_val("cluster_network_interface"); + if (networks.empty()) { + // fall back to public_ network and interface if cluster is not set + networks = cct->_conf->get_val("public_network"); + interfaces = + cct->_conf->get_val("public_network_interface"); + } + } + if (addr.is_blank_ip() && + !networks.empty()) { + // note: pass in ipv to filter the matching addresses + if ((ipv & CEPH_PICK_ADDRESS_IPV4) && + (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) { + fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4, networks, interfaces, + addrs); + } + if (ipv & CEPH_PICK_ADDRESS_IPV6) { + fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV6, networks, interfaces, + addrs); + } + if ((ipv & CEPH_PICK_ADDRESS_IPV4) && + !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) { + fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4, networks, interfaces, + addrs); + } + } + + // note: we may have a blank addr here + + // ipv4 and/or ipv6? + if (addrs->v.empty()) { + entity_addr_t addr; + addr.set_type(entity_addr_t::TYPE_MSGR2); + if ((ipv & CEPH_PICK_ADDRESS_IPV4) && + (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) { + addr.set_family(AF_INET); + addrs->v.push_back(addr); + } + if (ipv & CEPH_PICK_ADDRESS_IPV6) { + addr.set_family(AF_INET6); + addrs->v.push_back(addr); + } + if ((ipv & CEPH_PICK_ADDRESS_IPV4) && + !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) { + addr.set_family(AF_INET); + addrs->v.push_back(addr); + } + } + + // msgr2 or legacy or both? + if (msgrv == (CEPH_PICK_ADDRESS_MSGR1 | CEPH_PICK_ADDRESS_MSGR2)) { + vector v; + v.swap(addrs->v); + for (auto a : v) { + a.set_type(entity_addr_t::TYPE_MSGR2); + if (flags & CEPH_PICK_ADDRESS_DEFAULT_MON_PORTS) { + a.set_port(CEPH_MON_PORT_IANA); + } + addrs->v.push_back(a); + a.set_type(entity_addr_t::TYPE_LEGACY); + if (flags & CEPH_PICK_ADDRESS_DEFAULT_MON_PORTS) { + a.set_port(CEPH_MON_PORT_LEGACY); + } + addrs->v.push_back(a); + } + } else if (msgrv == CEPH_PICK_ADDRESS_MSGR1) { + for (auto& a : addrs->v) { + a.set_type(entity_addr_t::TYPE_LEGACY); + } + } else { + for (auto& a : addrs->v) { + a.set_type(entity_addr_t::TYPE_MSGR2); + } + } + + return 0; +} + +int pick_addresses( + CephContext *cct, + unsigned flags, + entity_addrvec_t *addrs) +{ + struct ifaddrs *ifa; + int r = getifaddrs(&ifa); + if (r < 0) { + r = -errno; + string err = cpp_strerror(r); + lderr(cct) << "unable to fetch interfaces and addresses: " + << cpp_strerror(r) << dendl; + return r; + } + r = pick_addresses(cct, flags, ifa, addrs); + freeifaddrs(ifa); + return r; +} std::string pick_iface(CephContext *cct, const struct sockaddr_storage &network) { @@ -227,8 +400,9 @@ std::string pick_iface(CephContext *cct, const struct sockaddr_storage &network) } const unsigned int prefix_len = max(sizeof(in_addr::s_addr), sizeof(in6_addr::s6_addr)) * CHAR_BIT; - const struct ifaddrs *found = find_ip_in_subnet(ifa, - (const struct sockaddr *) &network, prefix_len); + const struct ifaddrs *found = find_ip_in_subnet( + ifa, + (const struct sockaddr *) &network, prefix_len); std::string result; if (found) { diff --git a/src/common/pick_address.h b/src/common/pick_address.h index 6176473318612..70d78411477b9 100644 --- a/src/common/pick_address.h +++ b/src/common/pick_address.h @@ -4,6 +4,7 @@ #include "common/config.h" class CephContext; +class entity_addrvec_t; #define CEPH_PICK_ADDRESS_PUBLIC 0x01 @@ -36,6 +37,10 @@ class CephContext; */ void pick_addresses(CephContext *cct, int needs); +int pick_addresses(CephContext *cct, unsigned flags, entity_addrvec_t *addrs); +int pick_addresses(CephContext *cct, unsigned flags, struct ifaddrs *ifa, + entity_addrvec_t *addrs); + /** * Find a network interface whose address matches the address/netmask * in `network`. diff --git a/src/test/test_ipaddr.cc b/src/test/test_ipaddr.cc index 6db389325148e..36ff973870461 100644 --- a/src/test/test_ipaddr.cc +++ b/src/test/test_ipaddr.cc @@ -2,6 +2,8 @@ #include "common/pick_address.h" #include "global/global_context.h" #include "gtest/gtest.h" +#include "include/stringify.h" +#include "common/ceph_context.h" #if defined(__FreeBSD__) #include @@ -606,3 +608,198 @@ TEST(pick_address, find_ip_in_subnet_list) "eth1"); ASSERT_EQ((struct sockaddr*)&a_three, result); } + +TEST(pick_address, filtering) +{ + struct ifaddrs one, two, three; + struct sockaddr_in a_one; + struct sockaddr_in a_two; + struct sockaddr_in6 a_three; + + one.ifa_next = &two; + one.ifa_addr = (struct sockaddr*)&a_one; + one.ifa_name = eth0; + + two.ifa_next = &three; + two.ifa_addr = (struct sockaddr*)&a_two; + two.ifa_name = eth1; + + three.ifa_next = NULL; + three.ifa_addr = (struct sockaddr*)&a_three; + three.ifa_name = eth1; + + ipv4(&a_one, "10.1.1.2"); + ipv4(&a_two, "10.2.1.123"); + ipv6(&a_three, "2001:1234:5678:90ab::cdef"); + + CephContext *cct = new CephContext(CEPH_ENTITY_TYPE_MON); + cct->_conf->_clear_safe_to_start_threads(); // so we can set configs + + cct->_conf->set_val("public_addr", ""); + cct->_conf->set_val("public_network", ""); + cct->_conf->set_val("public_network_interface", ""); + cct->_conf->set_val("cluster_addr", ""); + cct->_conf->set_val("cluster_network", ""); + cct->_conf->set_val("cluster_network_interface", ""); + + entity_addrvec_t av; + { + int r = pick_addresses(cct, + CEPH_PICK_ADDRESS_PUBLIC | + CEPH_PICK_ADDRESS_IPV4 | + CEPH_PICK_ADDRESS_MSGR1, + &one, &av); + cout << av << std::endl; + ASSERT_EQ(0, r); + ASSERT_EQ(1, av.v.size()); + ASSERT_EQ(string("0.0.0.0:0/0"), stringify(av.v[0])); + } + { + int r = pick_addresses(cct, + CEPH_PICK_ADDRESS_PUBLIC | + CEPH_PICK_ADDRESS_IPV6 | + CEPH_PICK_ADDRESS_MSGR1, + &one, &av); + cout << av << std::endl; + ASSERT_EQ(0, r); + ASSERT_EQ(1, av.v.size()); + ASSERT_EQ(string("[::]:0/0"), stringify(av.v[0])); + } + { + cct->_conf->set_val("public_network", "10.2.0.0/16"); + int r = pick_addresses(cct, + CEPH_PICK_ADDRESS_PUBLIC | + CEPH_PICK_ADDRESS_IPV4 | + CEPH_PICK_ADDRESS_MSGR1, + &one, &av); + cout << av << std::endl; + ASSERT_EQ(0, r); + ASSERT_EQ(1, av.v.size()); + ASSERT_EQ(string("10.2.1.123:0/0"), stringify(av.v[0])); + cct->_conf->set_val("public_network", ""); + } + { + cct->_conf->set_val("public_network", "10.0.0.0/8"); + cct->_conf->set_val("public_network_interface", "eth1"); + int r = pick_addresses(cct, + CEPH_PICK_ADDRESS_PUBLIC | + CEPH_PICK_ADDRESS_IPV4 | + CEPH_PICK_ADDRESS_MSGR1, + &one, &av); + cout << av << std::endl; + ASSERT_EQ(0, r); + ASSERT_EQ(1, av.v.size()); + ASSERT_EQ(string("10.2.1.123:0/0"), stringify(av.v[0])); + cct->_conf->set_val("public_network", ""); + cct->_conf->set_val("public_network_interface", ""); + } + { + cct->_conf->set_val("public_network", "10.2.0.0/16"); + cct->_conf->set_val("cluster_network", "10.1.0.0/16"); + int r = pick_addresses(cct, + CEPH_PICK_ADDRESS_PUBLIC | + CEPH_PICK_ADDRESS_IPV4 | + CEPH_PICK_ADDRESS_MSGR1, + &one, &av); + cout << av << std::endl; + ASSERT_EQ(0, r); + ASSERT_EQ(1, av.v.size()); + ASSERT_EQ(string("10.2.1.123:0/0"), stringify(av.v[0])); + cct->_conf->set_val("public_network", ""); + cct->_conf->set_val("cluster_network", ""); + } + { + cct->_conf->set_val("public_network", "10.2.0.0/16"); + cct->_conf->set_val("cluster_network", "10.1.0.0/16"); + int r = pick_addresses(cct, + CEPH_PICK_ADDRESS_CLUSTER | + CEPH_PICK_ADDRESS_IPV4 | + CEPH_PICK_ADDRESS_MSGR1, + &one, &av); + cout << av << std::endl; + ASSERT_EQ(0, r); + ASSERT_EQ(1, av.v.size()); + ASSERT_EQ(string("10.1.1.2:0/0"), stringify(av.v[0])); + cct->_conf->set_val("public_network", ""); + cct->_conf->set_val("cluster_network", ""); + } + + { + cct->_conf->set_val("public_network", "2001::/16"); + int r = pick_addresses(cct, + CEPH_PICK_ADDRESS_PUBLIC | + CEPH_PICK_ADDRESS_IPV6 | + CEPH_PICK_ADDRESS_MSGR1, + &one, &av); + cout << av << std::endl; + ASSERT_EQ(0, r); + ASSERT_EQ(1, av.v.size()); + ASSERT_EQ(string("[2001:1234:5678:90ab::cdef]:0/0"), stringify(av.v[0])); + cct->_conf->set_val("public_network", ""); + } + { + cct->_conf->set_val("public_network", "2001::/16 10.0.0.0/8"); + cct->_conf->set_val("public_network_interface", "eth1"); + int r = pick_addresses(cct, + CEPH_PICK_ADDRESS_PUBLIC | + CEPH_PICK_ADDRESS_IPV4 | + CEPH_PICK_ADDRESS_IPV6 | + CEPH_PICK_ADDRESS_MSGR1, + &one, &av); + cout << av << std::endl; + ASSERT_EQ(0, r); + ASSERT_EQ(2, av.v.size()); + ASSERT_EQ(string("[2001:1234:5678:90ab::cdef]:0/0"), stringify(av.v[0])); + ASSERT_EQ(string("10.2.1.123:0/0"), stringify(av.v[1])); + cct->_conf->set_val("public_network", ""); + cct->_conf->set_val("public_network_interface", ""); + } + { + cct->_conf->set_val("public_network", "2001::/16 10.0.0.0/8"); + cct->_conf->set_val("public_network_interface", "eth1"); + int r = pick_addresses(cct, + CEPH_PICK_ADDRESS_PUBLIC | + CEPH_PICK_ADDRESS_IPV4 | + CEPH_PICK_ADDRESS_IPV6 | + CEPH_PICK_ADDRESS_MSGR1 | + CEPH_PICK_ADDRESS_PREFER_IPV4, + &one, &av); + cout << av << std::endl; + ASSERT_EQ(0, r); + ASSERT_EQ(2, av.v.size()); + ASSERT_EQ(string("10.2.1.123:0/0"), stringify(av.v[0])); + ASSERT_EQ(string("[2001:1234:5678:90ab::cdef]:0/0"), stringify(av.v[1])); + cct->_conf->set_val("public_network", ""); + cct->_conf->set_val("public_network_interface", ""); + } + + { + cct->_conf->set_val("public_network", "2001::/16"); + int r = pick_addresses(cct, + CEPH_PICK_ADDRESS_PUBLIC | + CEPH_PICK_ADDRESS_IPV6 | + CEPH_PICK_ADDRESS_MSGR1 | + CEPH_PICK_ADDRESS_MSGR2, + &one, &av); + cout << av << std::endl; + ASSERT_EQ(0, r); + ASSERT_EQ(2, av.v.size()); + ASSERT_EQ(string("msgr2:[2001:1234:5678:90ab::cdef]:0/0"), stringify(av.v[0])); + ASSERT_EQ(string("[2001:1234:5678:90ab::cdef]:0/0"), stringify(av.v[1])); + cct->_conf->set_val("public_network", ""); + } + + { + int r = pick_addresses(cct, + CEPH_PICK_ADDRESS_PUBLIC | + CEPH_PICK_ADDRESS_IPV4 | + CEPH_PICK_ADDRESS_MSGR1 | + CEPH_PICK_ADDRESS_MSGR2, + &one, &av); + cout << av << std::endl; + ASSERT_EQ(0, r); + ASSERT_EQ(2, av.v.size()); + ASSERT_EQ(string("msgr2:0.0.0.0:0/0"), stringify(av.v[0])); + ASSERT_EQ(string("0.0.0.0:0/0"), stringify(av.v[1])); + } +} -- 2.39.5