common/pick_address: get numa node info for a interface

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/ipaddr.cc b/src/common/ipaddr.cc
index 7e4112fbe56731de2b26a69cd833436690f82558..bceb32578af15a31c7e16a9e5b5347acc31bb195 100644 (file)
--- a/src/common/ipaddr.cc
+++ b/src/common/ipaddr.cc
@@ -11,6 +11,7 @@
 
 #include "include/ipaddr.h"
 #include "msg/msg_types.h"
+#include "common/pick_address.h"
 
 void netmask_ipv4(const struct in_addr *addr,
                         unsigned int prefix_len,
@@ -28,9 +29,20 @@ void netmask_ipv4(const struct in_addr *addr,
 }
 
 
+static bool match_numa_node(const string& if_name, int numa_node)
+{
+  int if_node = -1;
+  int r = get_iface_numa_node(if_name, &if_node);
+  if (r < 0) {
+    return false;
+  }
+  return if_node == numa_node;
+}
+
 const struct ifaddrs *find_ipv4_in_subnet(const struct ifaddrs *addrs,
-                                          const struct sockaddr_in *net,
-                                          unsigned int prefix_len) {
+                                         const struct sockaddr_in *net,
+                                         unsigned int prefix_len,
+                                         int numa_node) {
   struct in_addr want, temp;
 
   netmask_ipv4(&net->sin_addr, prefix_len, &want);
@@ -43,6 +55,9 @@ const struct ifaddrs *find_ipv4_in_subnet(const struct ifaddrs *addrs,
     if (strcmp(addrs->ifa_name, "lo") == 0)
       continue;
 
+    if (numa_node >= 0 && !match_numa_node(addrs->ifa_name, numa_node))
+      continue;
+
     if (addrs->ifa_addr->sa_family != net->sin_family)
       continue;
 
@@ -73,8 +88,9 @@ void netmask_ipv6(const struct in6_addr *addr,
 
 
 const struct ifaddrs *find_ipv6_in_subnet(const struct ifaddrs *addrs,
-                                          const struct sockaddr_in6 *net,
-                                          unsigned int prefix_len) {
+                                         const struct sockaddr_in6 *net,
+                                         unsigned int prefix_len,
+                                         int numa_node) {
   struct in6_addr want, temp;
 
   netmask_ipv6(&net->sin6_addr, prefix_len, &want);
@@ -87,6 +103,9 @@ const struct ifaddrs *find_ipv6_in_subnet(const struct ifaddrs *addrs,
     if (strcmp(addrs->ifa_name, "lo") == 0)
       continue;
 
+    if (numa_node >= 0 && !match_numa_node(addrs->ifa_name, numa_node))
+      continue;
+
     if (addrs->ifa_addr->sa_family != net->sin6_family)
       continue;
 
@@ -104,14 +123,17 @@ const struct ifaddrs *find_ipv6_in_subnet(const struct ifaddrs *addrs,
 
 
 const struct ifaddrs *find_ip_in_subnet(const struct ifaddrs *addrs,
-                                        const struct sockaddr *net,
-                                        unsigned int prefix_len) {
+                                       const struct sockaddr *net,
+                                       unsigned int prefix_len,
+                                       int numa_node) {
   switch (net->sa_family) {
     case AF_INET:
-      return find_ipv4_in_subnet(addrs, (struct sockaddr_in*)net, prefix_len);
+      return find_ipv4_in_subnet(addrs, (struct sockaddr_in*)net, prefix_len,
+                                numa_node);
 
     case AF_INET6:
-      return find_ipv6_in_subnet(addrs, (struct sockaddr_in6*)net, prefix_len);
+      return find_ipv6_in_subnet(addrs, (struct sockaddr_in6*)net, prefix_len,
+                                numa_node);
     }
 
   return NULL;

diff --git a/src/common/pick_address.cc b/src/common/pick_address.cc
index 90a0e0007dd583566de5b0ce9729905dd4d9d019..95b42436de23a84b493f533bab62bfa9b91554fc 100644 (file)
--- a/src/common/pick_address.cc
+++ b/src/common/pick_address.cc
@@ -18,6 +18,7 @@
 #include "common/config_obs.h"
 #include "common/debug.h"
 #include "common/errno.h"
+#include "common/numa.h"
 
 #include <netdb.h>
 
@@ -28,7 +29,8 @@ const struct sockaddr *find_ip_in_subnet_list(
   const struct ifaddrs *ifa,
   unsigned ipv,
   const std::string &networks,
-  const std::string &interfaces)
+  const std::string &interfaces,
+  int numa_node)
 {
   std::list<string> nets;
   get_str_list(networks, nets);
@@ -94,7 +96,7 @@ const struct sockaddr *find_ip_in_subnet_list(
 
     const struct ifaddrs *found = find_ip_in_subnet(
       filtered,
-      (struct sockaddr *) &net, prefix_len);
+      (struct sockaddr *) &net, prefix_len, numa_node);
     if (found) {
       r = found->ifa_addr;
       break;
@@ -133,14 +135,16 @@ static void fill_in_one_address(CephContext *cct,
                                const struct ifaddrs *ifa,
                                const string networks,
                                const string interfaces,
-                               const char *conf_var)
+                               const char *conf_var,
+                               int numa_node = -1)
 {
   const struct sockaddr *found = find_ip_in_subnet_list(
     cct,
     ifa,
     CEPH_PICK_ADDRESS_IPV4|CEPH_PICK_ADDRESS_IPV6,
     networks,
-    interfaces);
+    interfaces,
+    numa_node);
   if (!found) {
     lderr(cct) << "unable to find any IP address in networks '" << networks
               << "' interfaces '" << interfaces << "'" << dendl;
@@ -221,10 +225,11 @@ static int fill_in_one_address(
   unsigned ipv,
   const string networks,
   const string interfaces,
-  entity_addrvec_t *addrs)
+  entity_addrvec_t *addrs,
+  int numa_node = -1)
 {
   const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv, networks,
-                                                       interfaces);
+                                                       interfaces, numa_node);
   if (!found) {
     lderr(cct) << "unable to find any IP address in networks '" << networks
               << "' interfaces '" << interfaces << "'" << dendl;
@@ -261,7 +266,8 @@ int pick_addresses(
   CephContext *cct,
   unsigned flags,
   struct ifaddrs *ifa,
-  entity_addrvec_t *addrs)
+  entity_addrvec_t *addrs,
+  int preferred_numa_node)
 {
   addrs->v.clear();
 
@@ -327,29 +333,33 @@ int pick_addresses(
   }
   if (addr.is_blank_ip() &&
       !networks.empty()) {
-    // note: pass in ipv to filter the matching addresses
-    if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
-       (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
-      int r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4, networks, interfaces,
-                                 addrs);
-      if (r < 0) {
-       return r;
+    // first try on preferred numa node (if >= 0), then anywhere.
+    while (true) {
+      // note: pass in ipv to filter the matching addresses
+      int r = 0;
+      if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
+         (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
+       r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4, networks,
+                               interfaces, addrs, preferred_numa_node);
       }
-    }
-    if (ipv & CEPH_PICK_ADDRESS_IPV6) {
-      int r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV6, networks, interfaces,
-                                 addrs);
-      if (r < 0) {
-       return r;
+      if (r >= 0 &&
+         (ipv & CEPH_PICK_ADDRESS_IPV6)) {
+       r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV6, networks,
+                               interfaces, addrs, preferred_numa_node);
       }
-    }
-    if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
-       !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
-      int r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4, networks, interfaces,
-                                 addrs);
-      if (r < 0) {
+      if (r >= 0 &&
+         (ipv & CEPH_PICK_ADDRESS_IPV4) &&
+         !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
+       r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4, networks,
+                               interfaces, addrs, preferred_numa_node);
+      }
+      if (r >= 0) {
+       break;
+      }
+      if (preferred_numa_node < 0) {
        return r;
       }
+      preferred_numa_node = -1;      // try any numa node
     }
   }
 
@@ -407,7 +417,8 @@ int pick_addresses(
 int pick_addresses(
   CephContext *cct,
   unsigned flags,
-  entity_addrvec_t *addrs)
+  entity_addrvec_t *addrs,
+  int preferred_numa_node)
 {
   struct ifaddrs *ifa;
   int r = getifaddrs(&ifa);
@@ -418,7 +429,7 @@ int pick_addresses(
               <<  cpp_strerror(r) << dendl;
     return r;
   }
-  r = pick_addresses(cct, flags, ifa, addrs);
+  r = pick_addresses(cct, flags, ifa, addrs, preferred_numa_node);
   freeifaddrs(ifa);
   return r;
 }
@@ -477,3 +488,35 @@ bool have_local_addr(CephContext *cct, const list<entity_addr_t>& ls, entity_add
   freeifaddrs(ifa);
   return found;
 }
+
+int get_iface_numa_node(
+  const std::string& iface,
+  int *node)
+{
+  string fn = std::string("/sys/class/net/") + iface + "/device/numa_node";
+
+  int r = 0;
+  char buf[1024];
+  char *endptr = 0;
+  int fd = ::open(fn.c_str(), O_RDONLY);
+  if (fd < 0) {
+    return -errno;
+  }
+  r = safe_read(fd, &buf, sizeof(buf));
+  if (r < 0) {
+    goto out;
+  }
+  buf[r] = 0;
+  while (r > 0 && ::isspace(buf[--r])) {
+    buf[r] = 0;
+  }
+  *node = strtoll(buf, &endptr, 10);
+  if (endptr != buf + strlen(buf)) {
+    r = -EINVAL;
+    goto out;
+  }
+  r = 0;
+ out:
+  ::close(fd);
+  return r;
+}

diff --git a/src/common/pick_address.h b/src/common/pick_address.h
index 70d78411477b9821705e264b6b75e96a15762071..2eccfd360e90bac60cdcb9dae6f467483f4942d3 100644 (file)
--- a/src/common/pick_address.h
+++ b/src/common/pick_address.h
@@ -1,3 +1,5 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
 #ifndef CEPH_PICK_ADDRESS_H
 #define CEPH_PICK_ADDRESS_H
 
@@ -37,9 +39,11 @@ class entity_addrvec_t;
  */
 void pick_addresses(CephContext *cct, int needs);
 
-int pick_addresses(CephContext *cct, unsigned flags, entity_addrvec_t *addrs);
+int pick_addresses(CephContext *cct, unsigned flags, entity_addrvec_t *addrs,
+                  int preferred_numa_node = -1);
 int pick_addresses(CephContext *cct, unsigned flags, struct ifaddrs *ifa,
-                  entity_addrvec_t *addrs);
+                  entity_addrvec_t *addrs,
+                  int preferred_numa_node = -1);
 
 /**
  * Find a network interface whose address matches the address/netmask
@@ -64,6 +68,11 @@ const struct sockaddr *find_ip_in_subnet_list(
   const struct ifaddrs *ifa,
   unsigned ipv,
   const std::string &networks,
-  const std::string &interfaces);
+  const std::string &interfaces,
+  int numa_node=-1);
+
+int get_iface_numa_node(
+  const std::string& iface,
+  int *node);
 
 #endif

diff --git a/src/include/ipaddr.h b/src/include/ipaddr.h
index 2f72f40ef4b8dcea597de50a6713a31e91fe9dd3..e8bed82920af35a7db8dabeac776e38cf7714053 100644 (file)
--- a/src/include/ipaddr.h
+++ b/src/include/ipaddr.h
@@ -10,8 +10,9 @@ class entity_addr_t;
  * is system-dependent and should not be relied on.
  */
 const struct ifaddrs *find_ip_in_subnet(const struct ifaddrs *addrs,
-                                        const struct sockaddr *net,
-                                        unsigned int prefix_len);
+                                       const struct sockaddr *net,
+                                       unsigned int prefix_len,
+                                       int numa_node = -1);
 
 /*
  * Validate and parse IPv4 or IPv6 network