From a7c4419492e622c11aaa60d7e7e77f0115010cf6 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 3 Jan 2019 20:50:42 -0600 Subject: [PATCH] common/pick_address: get numa node info for a interface Signed-off-by: Sage Weil --- src/common/ipaddr.cc | 38 ++++++++++++--- src/common/pick_address.cc | 99 +++++++++++++++++++++++++++----------- src/common/pick_address.h | 15 ++++-- src/include/ipaddr.h | 5 +- 4 files changed, 116 insertions(+), 41 deletions(-) diff --git a/src/common/ipaddr.cc b/src/common/ipaddr.cc index 7e4112fbe5673..bceb32578af15 100644 --- a/src/common/ipaddr.cc +++ b/src/common/ipaddr.cc @@ -11,6 +11,7 @@ #include "include/ipaddr.h" #include "msg/msg_types.h" +#include "common/pick_address.h" void netmask_ipv4(const struct in_addr *addr, unsigned int prefix_len, @@ -28,9 +29,20 @@ void netmask_ipv4(const struct in_addr *addr, } +static bool match_numa_node(const string& if_name, int numa_node) +{ + int if_node = -1; + int r = get_iface_numa_node(if_name, &if_node); + if (r < 0) { + return false; + } + return if_node == numa_node; +} + const struct ifaddrs *find_ipv4_in_subnet(const struct ifaddrs *addrs, - const struct sockaddr_in *net, - unsigned int prefix_len) { + const struct sockaddr_in *net, + unsigned int prefix_len, + int numa_node) { struct in_addr want, temp; netmask_ipv4(&net->sin_addr, prefix_len, &want); @@ -43,6 +55,9 @@ const struct ifaddrs *find_ipv4_in_subnet(const struct ifaddrs *addrs, if (strcmp(addrs->ifa_name, "lo") == 0) continue; + if (numa_node >= 0 && !match_numa_node(addrs->ifa_name, numa_node)) + continue; + if (addrs->ifa_addr->sa_family != net->sin_family) continue; @@ -73,8 +88,9 @@ void netmask_ipv6(const struct in6_addr *addr, const struct ifaddrs *find_ipv6_in_subnet(const struct ifaddrs *addrs, - const struct sockaddr_in6 *net, - unsigned int prefix_len) { + const struct sockaddr_in6 *net, + unsigned int prefix_len, + int numa_node) { struct in6_addr want, temp; netmask_ipv6(&net->sin6_addr, prefix_len, &want); @@ -87,6 +103,9 @@ const struct ifaddrs *find_ipv6_in_subnet(const struct ifaddrs *addrs, if (strcmp(addrs->ifa_name, "lo") == 0) continue; + if (numa_node >= 0 && !match_numa_node(addrs->ifa_name, numa_node)) + continue; + if (addrs->ifa_addr->sa_family != net->sin6_family) continue; @@ -104,14 +123,17 @@ const struct ifaddrs *find_ipv6_in_subnet(const struct ifaddrs *addrs, const struct ifaddrs *find_ip_in_subnet(const struct ifaddrs *addrs, - const struct sockaddr *net, - unsigned int prefix_len) { + const struct sockaddr *net, + unsigned int prefix_len, + int numa_node) { switch (net->sa_family) { case AF_INET: - return find_ipv4_in_subnet(addrs, (struct sockaddr_in*)net, prefix_len); + return find_ipv4_in_subnet(addrs, (struct sockaddr_in*)net, prefix_len, + numa_node); case AF_INET6: - return find_ipv6_in_subnet(addrs, (struct sockaddr_in6*)net, prefix_len); + return find_ipv6_in_subnet(addrs, (struct sockaddr_in6*)net, prefix_len, + numa_node); } return NULL; diff --git a/src/common/pick_address.cc b/src/common/pick_address.cc index 90a0e0007dd58..95b42436de23a 100644 --- a/src/common/pick_address.cc +++ b/src/common/pick_address.cc @@ -18,6 +18,7 @@ #include "common/config_obs.h" #include "common/debug.h" #include "common/errno.h" +#include "common/numa.h" #include @@ -28,7 +29,8 @@ const struct sockaddr *find_ip_in_subnet_list( const struct ifaddrs *ifa, unsigned ipv, const std::string &networks, - const std::string &interfaces) + const std::string &interfaces, + int numa_node) { std::list nets; get_str_list(networks, nets); @@ -94,7 +96,7 @@ const struct sockaddr *find_ip_in_subnet_list( const struct ifaddrs *found = find_ip_in_subnet( filtered, - (struct sockaddr *) &net, prefix_len); + (struct sockaddr *) &net, prefix_len, numa_node); if (found) { r = found->ifa_addr; break; @@ -133,14 +135,16 @@ static void fill_in_one_address(CephContext *cct, const struct ifaddrs *ifa, const string networks, const string interfaces, - const char *conf_var) + const char *conf_var, + int numa_node = -1) { const struct sockaddr *found = find_ip_in_subnet_list( cct, ifa, CEPH_PICK_ADDRESS_IPV4|CEPH_PICK_ADDRESS_IPV6, networks, - interfaces); + interfaces, + numa_node); if (!found) { lderr(cct) << "unable to find any IP address in networks '" << networks << "' interfaces '" << interfaces << "'" << dendl; @@ -221,10 +225,11 @@ static int fill_in_one_address( unsigned ipv, const string networks, const string interfaces, - entity_addrvec_t *addrs) + entity_addrvec_t *addrs, + int numa_node = -1) { const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv, networks, - interfaces); + interfaces, numa_node); if (!found) { lderr(cct) << "unable to find any IP address in networks '" << networks << "' interfaces '" << interfaces << "'" << dendl; @@ -261,7 +266,8 @@ int pick_addresses( CephContext *cct, unsigned flags, struct ifaddrs *ifa, - entity_addrvec_t *addrs) + entity_addrvec_t *addrs, + int preferred_numa_node) { addrs->v.clear(); @@ -327,29 +333,33 @@ int pick_addresses( } if (addr.is_blank_ip() && !networks.empty()) { - // note: pass in ipv to filter the matching addresses - if ((ipv & CEPH_PICK_ADDRESS_IPV4) && - (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) { - int r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4, networks, interfaces, - addrs); - if (r < 0) { - return r; + // first try on preferred numa node (if >= 0), then anywhere. + while (true) { + // note: pass in ipv to filter the matching addresses + int r = 0; + if ((ipv & CEPH_PICK_ADDRESS_IPV4) && + (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) { + r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4, networks, + interfaces, addrs, preferred_numa_node); } - } - if (ipv & CEPH_PICK_ADDRESS_IPV6) { - int r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV6, networks, interfaces, - addrs); - if (r < 0) { - return r; + if (r >= 0 && + (ipv & CEPH_PICK_ADDRESS_IPV6)) { + r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV6, networks, + interfaces, addrs, preferred_numa_node); } - } - if ((ipv & CEPH_PICK_ADDRESS_IPV4) && - !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) { - int r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4, networks, interfaces, - addrs); - if (r < 0) { + if (r >= 0 && + (ipv & CEPH_PICK_ADDRESS_IPV4) && + !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) { + r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4, networks, + interfaces, addrs, preferred_numa_node); + } + if (r >= 0) { + break; + } + if (preferred_numa_node < 0) { return r; } + preferred_numa_node = -1; // try any numa node } } @@ -407,7 +417,8 @@ int pick_addresses( int pick_addresses( CephContext *cct, unsigned flags, - entity_addrvec_t *addrs) + entity_addrvec_t *addrs, + int preferred_numa_node) { struct ifaddrs *ifa; int r = getifaddrs(&ifa); @@ -418,7 +429,7 @@ int pick_addresses( << cpp_strerror(r) << dendl; return r; } - r = pick_addresses(cct, flags, ifa, addrs); + r = pick_addresses(cct, flags, ifa, addrs, preferred_numa_node); freeifaddrs(ifa); return r; } @@ -477,3 +488,35 @@ bool have_local_addr(CephContext *cct, const list& ls, entity_add freeifaddrs(ifa); return found; } + +int get_iface_numa_node( + const std::string& iface, + int *node) +{ + string fn = std::string("/sys/class/net/") + iface + "/device/numa_node"; + + int r = 0; + char buf[1024]; + char *endptr = 0; + int fd = ::open(fn.c_str(), O_RDONLY); + if (fd < 0) { + return -errno; + } + r = safe_read(fd, &buf, sizeof(buf)); + if (r < 0) { + goto out; + } + buf[r] = 0; + while (r > 0 && ::isspace(buf[--r])) { + buf[r] = 0; + } + *node = strtoll(buf, &endptr, 10); + if (endptr != buf + strlen(buf)) { + r = -EINVAL; + goto out; + } + r = 0; + out: + ::close(fd); + return r; +} diff --git a/src/common/pick_address.h b/src/common/pick_address.h index 70d78411477b9..2eccfd360e90b 100644 --- a/src/common/pick_address.h +++ b/src/common/pick_address.h @@ -1,3 +1,5 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab #ifndef CEPH_PICK_ADDRESS_H #define CEPH_PICK_ADDRESS_H @@ -37,9 +39,11 @@ class entity_addrvec_t; */ void pick_addresses(CephContext *cct, int needs); -int pick_addresses(CephContext *cct, unsigned flags, entity_addrvec_t *addrs); +int pick_addresses(CephContext *cct, unsigned flags, entity_addrvec_t *addrs, + int preferred_numa_node = -1); int pick_addresses(CephContext *cct, unsigned flags, struct ifaddrs *ifa, - entity_addrvec_t *addrs); + entity_addrvec_t *addrs, + int preferred_numa_node = -1); /** * Find a network interface whose address matches the address/netmask @@ -64,6 +68,11 @@ const struct sockaddr *find_ip_in_subnet_list( const struct ifaddrs *ifa, unsigned ipv, const std::string &networks, - const std::string &interfaces); + const std::string &interfaces, + int numa_node=-1); + +int get_iface_numa_node( + const std::string& iface, + int *node); #endif diff --git a/src/include/ipaddr.h b/src/include/ipaddr.h index 2f72f40ef4b8d..e8bed82920af3 100644 --- a/src/include/ipaddr.h +++ b/src/include/ipaddr.h @@ -10,8 +10,9 @@ class entity_addr_t; * is system-dependent and should not be relied on. */ const struct ifaddrs *find_ip_in_subnet(const struct ifaddrs *addrs, - const struct sockaddr *net, - unsigned int prefix_len); + const struct sockaddr *net, + unsigned int prefix_len, + int numa_node = -1); /* * Validate and parse IPv4 or IPv6 network -- 2.39.5