krbd: add support for msgr2

Recognize ms_mode map option and filter initial monitor addresses
accordingly: if ms_mode is not given or ms_mode=legacy, discard v2
addresses, otherwise discard v1 addresses.

Note that nothing was discarded (i.e. v2 addresses were passed to
the kernel) previously.  The intent was to preserve that behaviour
in case ms_mode is not given, allowing to change the kernel default
in the future.  However, it turns out that mount.ceph helper has
been misguidedly discarding v2 addresses since commit eae01275134e
("mount.ceph: fork a child to get info from local configuration"),
so that ship has sailed.

Fixes: https://tracker.ceph.com/issues/48976
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>

diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst
index 94cdd0bfdde17bdf6230d64b56b196815407f5f0..92307e24ac0d9c9292822296717175126f91148b 100644 (file)
--- a/doc/man/8/rbd.rst
+++ b/doc/man/8/rbd.rst
@@ -744,9 +744,13 @@ Per client instance `rbd device map` options:
 
 * noshare - Disable sharing of client instances with other mappings.
 
-* crc - Enable CRC32C checksumming for data writes (default).
+* crc - Enable CRC32C checksumming for msgr1 on-the-wire protocol (default).
+  For msgr2.1 protocol this option is ignored: full checksumming is always on
+  in 'crc' mode and always off in 'secure' mode.
 
-* nocrc - Disable CRC32C checksumming for data writes.
+* nocrc - Disable CRC32C checksumming for msgr1 on-the-wire protocol.  Note
+  that only payload checksumming is disabled, header checksumming is always on.
+  For msgr2.1 protocol this option is ignored.
 
 * cephx_require_signatures - Require cephx message signing (since 3.19,
   default).
@@ -759,9 +763,12 @@ Per client instance `rbd device map` options:
 
 * notcp_nodelay - Enable Nagle's algorithm on client sockets (since 4.0).
 
-* cephx_sign_messages - Enable message signing (since 4.4, default).
+* cephx_sign_messages - Enable message signing for msgr1 on-the-wire protocol
+  (since 4.4, default).  For msgr2.1 protocol this option is ignored: message
+  signing is built into 'secure' mode and not offered in 'crc' mode.
 
-* nocephx_sign_messages - Disable message signing (since 4.4).
+* nocephx_sign_messages - Disable message signing for msgr1 on-the-wire protocol
+  (since 4.4).  For msgr2.1 protocol this option is ignored.
 
 * mount_timeout=x - A timeout on various steps in `rbd device map` and
   `rbd device unmap` sequences (default is 60 seconds).  In particular,
@@ -856,6 +863,25 @@ Per mapping (block device) `rbd device map` options:
   backend that the data is incompressible, disabling compression in aggressive
   mode (since 5.8).
 
+* ms_mode=legacy - Use msgr1 on-the-wire protocol (since 5.11, default).
+
+* ms_mode=crc - Use msgr2.1 on-the-wire protocol, select 'crc' mode, also
+  referred to as plain mode (since 5.11).  If the daemon denies 'crc' mode,
+  fail the connection.
+
+* ms_mode=secure - Use msgr2.1 on-the-wire protocol, select 'secure' mode
+  (since 5.11).  'secure' mode provides full in-transit encryption ensuring
+  both confidentiality and authenticity.  If the daemon denies 'secure' mode,
+  fail the connection.
+
+* ms_mode=prefer-crc - Use msgr2.1 on-the-wire protocol, select 'crc'
+  mode (since 5.11).  If the daemon denies 'crc' mode in favor of 'secure'
+  mode, agree to 'secure' mode.
+
+* ms_mode=prefer-secure - Use msgr2.1 on-the-wire protocol, select 'secure'
+  mode (since 5.11).  If the daemon denies 'secure' mode in favor of 'crc'
+  mode, agree to 'crc' mode.
+
 * udev - Wait for udev device manager to finish executing all matching
   "add" rules and release the device before exiting (default).  This option
   is not passed to the kernel.

diff --git a/src/krbd.cc b/src/krbd.cc
index 0d463dea52e92dfa7085044e32afb657d8ed054e..5e011d7a9a077becce2b83cfac72f8390948ec1d 100644 (file)
--- a/src/krbd.cc
+++ b/src/krbd.cc
@@ -44,6 +44,7 @@
 
 #include <blkid/blkid.h>
 #include <boost/algorithm/string/predicate.hpp>
+#include <boost/tokenizer.hpp>
 #include <libudev.h>
 
 static const int UDEV_BUF_SIZE = 1 << 20;  /* doubled to 2M (SO_RCVBUFFORCE) */
@@ -184,24 +185,39 @@ static int have_minor_attr(void)
 }
 
 static int build_map_buf(CephContext *cct, const krbd_spec& spec,
-                         const char *options, string *pbuf)
+                         const string& options, string *pbuf)
 {
+  bool msgr2 = false;
   ostringstream oss;
   int r;
 
+  boost::char_separator<char> sep(",");
+  boost::tokenizer<boost::char_separator<char>> tok(options, sep);
+  for (const auto& t : tok) {
+    if (boost::starts_with(t, "ms_mode=")) {
+      /* msgr2 unless ms_mode=legacy */
+      msgr2 = t.compare(8, t.npos, "legacy");
+    }
+  }
+
   MonMap monmap;
   r = monmap.build_initial(cct, false, cerr);
   if (r < 0)
     return r;
 
-  list<entity_addr_t> mon_addr;
-  monmap.list_addrs(mon_addr);
-
-  for (const auto &p : mon_addr) {
-    if (oss.tellp() > 0) {
-      oss << ",";
+  /*
+   * If msgr2, filter TYPE_MSGR2 addresses.  Otherwise, filter
+   * TYPE_LEGACY addresses.
+   */
+  for (const auto& p : monmap.mon_info) {
+    for (const auto& a : p.second.public_addrs.v) {
+      if ((msgr2 && a.is_msgr2()) || (!msgr2 && a.is_legacy())) {
+        if (oss.tellp() > 0) {
+          oss << ",";
+        }
+        oss << a.get_sockaddr();
+      }
     }
-    oss << p.get_sockaddr();
   }
 
   oss << " name=" << cct->_conf->name.get_id();
@@ -244,7 +260,7 @@ static int build_map_buf(CephContext *cct, const krbd_spec& spec,
     oss << ",key=" << key_name;
   }
 
-  if (strcmp(options, "") != 0)
+  if (!options.empty())
     oss << "," << options;
   if (!spec.nspace_name.empty())
     oss << ",_pool_ns=" << spec.nspace_name;

diff --git a/src/tools/rbd/action/Kernel.cc b/src/tools/rbd/action/Kernel.cc
index deddcea3d01b55d5d6fc2f7d97e7aea1faa13a32..3ed8e59037b0950042971587369030cb6ea1d6aa 100644 (file)
--- a/src/tools/rbd/action/Kernel.cc
+++ b/src/tools/rbd/action/Kernel.cc
@@ -83,6 +83,16 @@ static std::string map_option_compression_hint_cb(const char *value_char)
   return "";
 }
 
+static std::string map_option_ms_mode_cb(const char *value_char)
+{
+  if (!strcmp(value_char, "legacy") || !strcmp(value_char, "crc") ||
+      !strcmp(value_char, "secure") || !strcmp(value_char, "prefer-crc") ||
+      !strcmp(value_char, "prefer-secure")) {
+    return value_char;
+  }
+  return "";
+}
+
 static void put_map_option(const std::string &key, const std::string &val,
                            MapOptions* map_options)
 {
@@ -196,6 +206,10 @@ static int parse_map_options(const std::string &options_string,
       if (put_map_option_value("compression_hint", value_char,
                                map_option_compression_hint_cb, map_options))
         return -EINVAL;
+    } else if (!strcmp(this_char, "ms_mode")) {
+      if (put_map_option_value("ms_mode", value_char, map_option_ms_mode_cb,
+                               map_options))
+        return -EINVAL;
     } else if (!strcmp(this_char, "udev") || !strcmp(this_char, "noudev")) {
       put_map_option("udev", this_char, map_options);
     } else {