rbd nbd: add support for netlink interface

This adds support for mapping and unmapping using the netlink
interface. If the user passes in --try-netlink during mapping then we
will try to use the netlink interface and if that is not supported use
the ioctl interface. For unmap we will always try netlink and if that is
not supported drop down to the ioctl.

Signed-off-by: Mike Christie <mchristi@redhat.com>

diff --git a/src/tools/rbd_nbd/nbd-netlink.h b/src/tools/rbd_nbd/nbd-netlink.h
new file mode 100644 (file)
index 0000000..fd0f4e4
--- /dev/null
+++ b/src/tools/rbd_nbd/nbd-netlink.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2017 Facebook.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+#ifndef _UAPILINUX_NBD_NETLINK_H
+#define _UAPILINUX_NBD_NETLINK_H
+
+#define NBD_GENL_FAMILY_NAME   "nbd"
+#define NBD_GENL_VERSION       0x1
+
+/* Configuration policy attributes, used for CONNECT */
+enum {
+       NBD_ATTR_UNSPEC,
+       NBD_ATTR_INDEX,
+       NBD_ATTR_SIZE_BYTES,
+       NBD_ATTR_BLOCK_SIZE_BYTES,
+       NBD_ATTR_TIMEOUT,
+       NBD_ATTR_SERVER_FLAGS,
+       NBD_ATTR_CLIENT_FLAGS,
+       NBD_ATTR_SOCKETS,
+       __NBD_ATTR_MAX,
+};
+#define NBD_ATTR_MAX (__NBD_ATTR_MAX - 1)
+
+/*
+ * This is the format for multiple sockets with NBD_ATTR_SOCKETS
+ *
+ * [NBD_ATTR_SOCKETS]
+ *   [NBD_SOCK_ITEM]
+ *     [NBD_SOCK_FD]
+ *   [NBD_SOCK_ITEM]
+ *     [NBD_SOCK_FD]
+ */
+enum {
+       NBD_SOCK_ITEM_UNSPEC,
+       NBD_SOCK_ITEM,
+       __NBD_SOCK_ITEM_MAX,
+};
+#define NBD_SOCK_ITEM_MAX (__NBD_SOCK_ITEM_MAX - 1)
+
+enum {
+       NBD_SOCK_UNSPEC,
+       NBD_SOCK_FD,
+       __NBD_SOCK_MAX,
+};
+#define NBD_SOCK_MAX (__NBD_SOCK_MAX - 1)
+
+enum {
+       NBD_CMD_UNSPEC,
+       NBD_CMD_CONNECT,
+       NBD_CMD_DISCONNECT,
+       __NBD_CMD_MAX,
+};
+#define NBD_CMD_MAX    (__NBD_CMD_MAX - 1)
+
+#endif /* _UAPILINUX_NBD_NETLINK_H */

diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc
index 9919a0dfdacd078a9ee0a0855b06bc1cc97446d7..5740cf4560e59079a150a5e11ca14d134202d4d0 100644 (file)
--- a/src/tools/rbd_nbd/rbd-nbd.cc
+++ b/src/tools/rbd_nbd/rbd-nbd.cc
@@ -32,6 +32,11 @@
 #include <sys/ioctl.h>
 #include <sys/socket.h>
 
+#include "nbd-netlink.h"
+#include <libnl3/netlink/genl/genl.h>
+#include <libnl3/netlink/genl/ctrl.h>
+#include <libnl3/netlink/genl/mngt.h>
+
 #include <fstream>
 #include <iostream>
 #include <memory>
@@ -72,6 +77,7 @@ struct Config {
   bool exclusive = false;
   bool readonly = false;
   bool set_max_part = false;
+  bool try_netlink = false;
 
   std::string poolname;
   std::string nsname;
@@ -95,6 +101,7 @@ static void usage()
             << "  --max_part <limit>      Override for module param max_part\n"
             << "  --exclusive             Forbid writes by other clients\n"
             << "  --timeout <seconds>     Set nbd request timeout\n"
+            << "  --try-netlink           Use the nbd netlink interface\n"
             << "\n"
             << "List options:\n"
             << "  --format plain|json|xml Output format (default: plain)\n"
@@ -104,6 +111,7 @@ static void usage()
 }
 
 static int nbd = -1;
+static int nbd_index = -1;
 
 enum Command {
   None,
@@ -665,8 +673,7 @@ static int parse_nbd_index(const std::string& devpath)
   return index;
 }
 
-static int try_ioctl_setup(Config *cfg, int fd, uint64_t size, int *nbd_index,
-                           uint64_t flags)
+static int try_ioctl_setup(Config *cfg, int fd, uint64_t size, uint64_t flags)
 {
   int index = 0, r;
 
@@ -755,7 +762,8 @@ static int try_ioctl_setup(Config *cfg, int fd, uint64_t size, int *nbd_index,
     }
   }
 
-  *nbd_index = index;
+  dout(10) << "ioctl setup complete for " << cfg->devpath << dendl;
+  nbd_index = index;
   return 0;
 
 close_nbd:
@@ -768,27 +776,255 @@ done:
   return r;
 }
 
+static void netlink_cleanup(struct nl_sock *sock)
+{
+  if (!sock)
+    return;
+
+  nl_close(sock);
+  nl_socket_free(sock);
+}
+
+static struct nl_sock *netlink_init(int *id)
+{
+  struct nl_sock *sock;
+  int ret;
+
+  sock = nl_socket_alloc();
+  if (!sock) {
+    cerr << "rbd-nbd: Could not allocate netlink socket." << std::endl;
+    return NULL;
+  }
+
+  ret = genl_connect(sock);
+  if (ret < 0) {
+    cerr << "rbd-nbd: Could not connect netlink socket. Error " << ret
+         << std::endl;
+    goto free_sock;
+  }
+
+  *id = genl_ctrl_resolve(sock, "nbd");
+  if (*id < 0)
+    //  nbd netlink interface not supported.
+    goto close_sock;
+
+  return sock;
+
+close_sock:
+  nl_close(sock);
+free_sock:
+  nl_socket_free(sock);
+  return NULL;
+}
+
+static int netlink_disconnect(int index)
+{
+  struct nl_sock *sock;
+  struct nl_msg *msg;
+  int ret, nl_id;
+
+  sock = netlink_init(&nl_id);
+  if (!sock)
+    // Try ioctl
+    return 1;
+
+  nl_socket_modify_cb(sock, NL_CB_VALID, NL_CB_CUSTOM, genl_handle_msg, NULL);
+
+  msg = nlmsg_alloc();
+  if (!msg) {
+    cerr << "rbd-nbd: Could not allocate netlink message." << std::endl;
+    goto free_sock;
+  }
+
+  if (!genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, nl_id, 0, 0,
+                   NBD_CMD_DISCONNECT, 0)) {
+    cerr << "rbd-nbd: Could not setup message." << std::endl;
+    goto nla_put_failure;
+  }
+
+  NLA_PUT_U32(msg, NBD_ATTR_INDEX, index);
+
+  ret = nl_send_sync(sock, msg);
+  netlink_cleanup(sock);
+  if (ret < 0) {
+    cerr << "rbd-nbd: netlink disconnect failed: " << nl_geterror(-ret)
+         << std::endl;
+    return -EIO;
+  }
+
+  return 0;
+
+nla_put_failure:
+  nlmsg_free(msg);
+free_sock:
+  netlink_cleanup(sock);
+  return -EIO;
+}
+
+static int netlink_disconnect_by_path(const std::string& devpath)
+{
+  int index;
+
+  index = parse_nbd_index(devpath);
+  if (index < 0)
+    return index;
+
+  return netlink_disconnect(index);
+}
+
+static int netlink_connect_cb(struct nl_msg *msg, void *arg)
+{
+  struct genlmsghdr *gnlh = (struct genlmsghdr *)nlmsg_data(nlmsg_hdr(msg));
+  Config *cfg = (Config *)arg;
+  struct nlattr *msg_attr[NBD_ATTR_MAX + 1];
+  uint32_t index;
+  int ret;
+
+  ret = nla_parse(msg_attr, NBD_ATTR_MAX, genlmsg_attrdata(gnlh, 0),
+                  genlmsg_attrlen(gnlh, 0), NULL);
+  if (ret) {
+    cerr << "rbd-nbd: Unsupported netlink reply" << std::endl;
+    return -NLE_MSGTYPE_NOSUPPORT;
+  }
+
+  if (!msg_attr[NBD_ATTR_INDEX]) {
+    cerr << "rbd-nbd: netlink connect reply missing device index." << std::endl;
+    return -NLE_MSGTYPE_NOSUPPORT;
+  }
+
+  index = nla_get_u32(msg_attr[NBD_ATTR_INDEX]);
+  cfg->devpath = "/dev/nbd" + stringify(index);
+  nbd_index = index;
+
+  return NL_OK;
+}
+
+static int netlink_connect(Config *cfg, struct nl_sock *sock, int nl_id, int fd,
+                           uint64_t size, uint64_t flags)
+{
+  struct nlattr *sock_attr;
+  struct nlattr *sock_opt;
+  struct nl_msg *msg;
+  int ret;
+
+  nl_socket_modify_cb(sock, NL_CB_VALID, NL_CB_CUSTOM, netlink_connect_cb, cfg);
+
+  msg = nlmsg_alloc();
+  if (!msg) {
+    cerr << "rbd-nbd: Could not allocate netlink message." << std::endl;
+    return -ENOMEM;
+  }
+
+  if (!genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, nl_id, 0, 0, NBD_CMD_CONNECT,
+                   0)) {
+    cerr << "rbd-nbd: Could not setup message." << std::endl;
+    goto free_msg;
+  }
+
+  if (!cfg->devpath.empty()) {
+    ret = parse_nbd_index(cfg->devpath);
+    if (ret < 0)
+      goto free_msg;
+
+    NLA_PUT_U32(msg, NBD_ATTR_INDEX, ret);
+  }
+
+  if (cfg->timeout >= 0)
+    NLA_PUT_U64(msg, NBD_ATTR_TIMEOUT, cfg->timeout);
+
+  NLA_PUT_U64(msg, NBD_ATTR_SIZE_BYTES, size);
+  NLA_PUT_U64(msg, NBD_ATTR_BLOCK_SIZE_BYTES, RBD_NBD_BLKSIZE);
+  NLA_PUT_U64(msg, NBD_ATTR_SERVER_FLAGS, flags);
+
+  sock_attr = nla_nest_start(msg, NBD_ATTR_SOCKETS);
+  if (!sock_attr) {
+    cerr << "rbd-nbd: Could not init sockets in netlink message." << std::endl;
+    goto free_msg;
+  }
+
+  sock_opt = nla_nest_start(msg, NBD_SOCK_ITEM);
+  if (!sock_opt) {
+    cerr << "rbd-nbd: Could not init sock in netlink message." << std::endl;
+    goto free_msg;
+  }
+
+  NLA_PUT_U32(msg, NBD_SOCK_FD, fd);
+  nla_nest_end(msg, sock_opt);
+  nla_nest_end(msg, sock_attr);
+
+  ret = nl_send_sync(sock, msg);
+  if (ret < 0) {
+    cerr << "rbd-nbd: netlink connect failed: " << nl_geterror(ret)
+         << std::endl;
+    return -EIO;
+  }
+
+  dout(10) << "netlink connect complete for " << cfg->devpath << dendl;
+  return 0;
+
+nla_put_failure:
+free_msg:
+  nlmsg_free(msg);
+  return -EIO;
+}
+
+static int try_netlink_setup(Config *cfg, int fd, uint64_t size, uint64_t flags)
+{
+  struct nl_sock *sock;
+  int nl_id, ret;
+
+  sock = netlink_init(&nl_id);
+  if (!sock) {
+    cerr << "rbd-nbd: Netlink interface not supported. Using ioctl interface."
+         << std::endl;
+    return 1;
+  }
+
+  dout(10) << "netlink interface supported." << dendl;
+
+  ret = netlink_connect(cfg, sock, nl_id, fd, size, flags);
+  netlink_cleanup(sock);
+
+  if (ret != 0)
+    return ret;
+
+  nbd = open_device(cfg->devpath.c_str());
+  if (nbd < 0) {
+    cerr << "rbd-nbd: failed to open device: " << cfg->devpath << std::endl;
+    return nbd;
+  }
+
+  return 0;
+}
+
 static void handle_signal(int signum)
 {
+  int ret;
+
   ceph_assert(signum == SIGINT || signum == SIGTERM);
   derr << "*** Got signal " << sig_str(signum) << " ***" << dendl;
+
+  if (nbd < 0 || nbd_index < 0) {
+    dout(20) << __func__ << ": " << "disconnect not needed." << dendl;
+    return;
+  }
+
   dout(20) << __func__ << ": " << "sending NBD_DISCONNECT" << dendl;
-  if (ioctl(nbd, NBD_DISCONNECT) < 0) {
-    derr << "rbd-nbd: disconnect failed: " << cpp_strerror(errno) << dendl;
+  ret = netlink_disconnect(nbd_index);
+  if (ret == 1)
+    ret = ioctl(nbd, NBD_DISCONNECT);
+
+  if (ret != 0) {
+    derr << "rbd-nbd: disconnect failed. Error: " << ret << dendl;
   } else {
     dout(20) << __func__ << ": " << "disconnected" << dendl;
   }
 }
 
-static NBDServer *start_server(Preforker& forker, int fd, librbd::Image& image)
+static NBDServer *start_server(int fd, librbd::Image& image)
 {
   NBDServer *server;
 
-  if (g_conf()->daemonize) {
-    global_init_postfork_finish(g_ceph_context);
-    forker.daemonize();
-  }
-
   server = new NBDServer(fd, image);
   server->start();
 
@@ -800,9 +1036,17 @@ static NBDServer *start_server(Preforker& forker, int fd, librbd::Image& image)
   return server;
 }
 
-static void run_server(void)
+static void run_server(Preforker& forker, NBDServer *server, bool netlink_used)
 {
-  ioctl(nbd, NBD_DO_IT);
+  if (g_conf()->daemonize) {
+    global_init_postfork_finish(g_ceph_context);
+    forker.daemonize();
+  }
+
+  if (netlink_used)
+    server->wait_for_disconnect();
+  else
+    ioctl(nbd, NBD_DO_IT);
 
   unregister_async_signal_handler(SIGHUP, sighup_handler);
   unregister_async_signal_handler(SIGINT, handle_signal);
@@ -822,8 +1066,8 @@ static int do_map(int argc, const char *argv[], Config *cfg)
   int read_only = 0;
   unsigned long flags;
   unsigned long size;
+  bool use_netlink;
 
-  int nbd_index;
   int fd[2];
 
   librbd::image_info_t info;
@@ -923,14 +1167,27 @@ static int do_map(int argc, const char *argv[], Config *cfg)
 
   size = info.size;
 
-  r = try_ioctl_setup(cfg, fd[0], size, &nbd_index, flags);
-  if (r < 0)
-    goto close_fd;
+  server = start_server(fd[1], image);
+
+  use_netlink = cfg->try_netlink;
+  if (use_netlink) {
+    r = try_netlink_setup(cfg, fd[0], size, flags);
+    if (r < 0) {
+      goto free_server;
+    } else if (r == 1) {
+      use_netlink = false;
+    }
+  }
+
+  if (!use_netlink) {
+    r = try_ioctl_setup(cfg, fd[0], size, flags);
+    if (r < 0)
+      goto free_server;
+  }
 
   r = check_device_size(nbd_index, size);
-  if (r < 0) {
+  if (r < 0)
     goto close_nbd;
-  }
 
   r = ioctl(nbd, BLKROSET, (unsigned long) &read_only);
   if (r < 0) {
@@ -948,20 +1205,25 @@ static int do_map(int argc, const char *argv[], Config *cfg)
 
     cout << cfg->devpath << std::endl;
 
-    server = start_server(forker, fd[1], image);
-    run_server();
+    run_server(forker, server, use_netlink);
 
     r = image.update_unwatch(handle);
     ceph_assert(r == 0);
   }
 
-  delete server;
 close_nbd:
   if (r < 0) {
-    ioctl(nbd, NBD_CLEAR_SOCK);
-    cerr << "rbd-nbd: failed to map, status: " << cpp_strerror(-r) << std::endl;
+    if (use_netlink) {
+      netlink_disconnect(nbd_index);
+    } else {
+      ioctl(nbd, NBD_CLEAR_SOCK);
+      cerr << "rbd-nbd: failed to map, status: " << cpp_strerror(-r)
+          << std::endl;
+    }
   }
   close(nbd);
+free_server:
+  delete server;
 close_fd:
   close(fd[0]);
   close(fd[1]);
@@ -975,23 +1237,30 @@ close_ret:
   return r;
 }
 
-static int do_unmap(const std::string &devpath)
+static int do_unmap(Config *cfg)
 {
-  int r = 0;
+  int r, nbd;
+
+  /*
+   * The netlink disconnect call supports devices setup with netlink or ioctl,
+   * so we always try that first.
+   */
+  r = netlink_disconnect_by_path(cfg->devpath);
+  if (r != 1)
+    return r;
 
-  int nbd = open_device(devpath.c_str());
+  nbd = open_device(cfg->devpath.c_str());
   if (nbd < 0) {
-    cerr << "rbd-nbd: failed to open device: " << devpath << std::endl;
+    cerr << "rbd-nbd: failed to open device: " << cfg->devpath << std::endl;
     return nbd;
   }
 
   r = ioctl(nbd, NBD_DISCONNECT);
   if (r < 0) {
-      cerr << "rbd-nbd: the device is not used" << std::endl; 
+      cerr << "rbd-nbd: the device is not used" << std::endl;
   }
 
   close(nbd);
-
   return r;
 }
 
@@ -1160,6 +1429,8 @@ static int parse_args(vector<const char*>& args, std::ostream *err_msg,
                                      (char *)NULL)) {
     } else if (ceph_argparse_flag(args, i, "--pretty-format", (char *)NULL)) {
       cfg->pretty_format = true;
+    } else if (ceph_argparse_flag(args, i, "--try-netlink", (char *)NULL)) {
+      cfg->try_netlink = true;
     } else {
       ++i;
     }
@@ -1260,7 +1531,7 @@ static int rbd_nbd(int argc, const char *argv[])
         return -EINVAL;
       break;
     case Disconnect:
-      r = do_unmap(cfg.devpath);
+      r = do_unmap(&cfg);
       if (r < 0)
         return -EINVAL;
       break;