krbd: add support for msgr2

Recognize ms_mode map option and filter initial monitor addresses
accordingly: if ms_mode is not given or ms_mode=legacy, discard v2
addresses, otherwise discard v1 addresses.

Note that nothing was discarded (i.e. v2 addresses were passed to
the kernel) previously.  The intent was to preserve that behaviour
in case ms_mode is not given, allowing to change the kernel default
in the future.  However, it turns out that mount.ceph helper has
been misguidedly discarding v2 addresses since commit eae01275134e
("mount.ceph: fork a child to get info from local configuration"),
so that ship has sailed.

Fixes: https://tracker.ceph.com/issues/48976
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
(cherry picked from commit 08f714964b7fe5024504818f01328a41acc24965)

Conflicts:
	doc/man/8/rbd.rst [ crush_location, read_from_replica and
	  compression_hint map options not in nautilus ]
	src/krbd.cc [ commit a1aecba6f44d ("krbd: make wait_for_udev_add()
	  clearer and a bit more efficient") not in nautilus ]
	src/tools/rbd/action/Kernel.cc [ commit 34f539d8af33 ("rbd:
	  delay parsing of default kernel map options") not in nautilus ]

diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst
index a3222661b930702858d5c7e350c42d9bdeae1f78..00ebeeb435d8f5912b72ab0ea795b61072e46056 100644 (file)
--- a/doc/man/8/rbd.rst
+++ b/doc/man/8/rbd.rst
@@ -704,9 +704,13 @@ Per client instance `rbd device map` options:
 
 * noshare - Disable sharing of client instances with other mappings.
 
-* crc - Enable CRC32C checksumming for data writes (default).
+* crc - Enable CRC32C checksumming for msgr1 on-the-wire protocol (default).
+  For msgr2.1 protocol this option is ignored: full checksumming is always on
+  in 'crc' mode and always off in 'secure' mode.
 
-* nocrc - Disable CRC32C checksumming for data writes.
+* nocrc - Disable CRC32C checksumming for msgr1 on-the-wire protocol.  Note
+  that only payload checksumming is disabled, header checksumming is always on.
+  For msgr2.1 protocol this option is ignored.
 
 * cephx_require_signatures - Require cephx message signing (since 3.19,
   default).
@@ -719,9 +723,12 @@ Per client instance `rbd device map` options:
 
 * notcp_nodelay - Enable Nagle's algorithm on client sockets (since 4.0).
 
-* cephx_sign_messages - Enable message signing (since 4.4, default).
+* cephx_sign_messages - Enable message signing for msgr1 on-the-wire protocol
+  (since 4.4, default).  For msgr2.1 protocol this option is ignored: message
+  signing is built into 'secure' mode and not offered in 'crc' mode.
 
-* nocephx_sign_messages - Disable message signing (since 4.4).
+* nocephx_sign_messages - Disable message signing for msgr1 on-the-wire protocol
+  (since 4.4).  For msgr2.1 protocol this option is ignored.
 
 * mount_timeout=x - A timeout on various steps in `rbd device map` and
   `rbd device unmap` sequences (default is 60 seconds).  In particular,
@@ -765,6 +772,25 @@ Per mapping (block device) `rbd device map` options:
   solid-state drives).  For filestore with filestore_punch_hole = false, the
   recommended setting is image object size (typically 4M).
 
+* ms_mode=legacy - Use msgr1 on-the-wire protocol (since 5.11, default).
+
+* ms_mode=crc - Use msgr2.1 on-the-wire protocol, select 'crc' mode, also
+  referred to as plain mode (since 5.11).  If the daemon denies 'crc' mode,
+  fail the connection.
+
+* ms_mode=secure - Use msgr2.1 on-the-wire protocol, select 'secure' mode
+  (since 5.11).  'secure' mode provides full in-transit encryption ensuring
+  both confidentiality and authenticity.  If the daemon denies 'secure' mode,
+  fail the connection.
+
+* ms_mode=prefer-crc - Use msgr2.1 on-the-wire protocol, select 'crc'
+  mode (since 5.11).  If the daemon denies 'crc' mode in favor of 'secure'
+  mode, agree to 'secure' mode.
+
+* ms_mode=prefer-secure - Use msgr2.1 on-the-wire protocol, select 'secure'
+  mode (since 5.11).  If the daemon denies 'secure' mode in favor of 'crc'
+  mode, agree to 'crc' mode.
+
 * udev - Wait for udev device manager to finish executing all matching
   "add" rules and release the device before exiting (default).  This option
   is not passed to the kernel.

diff --git a/src/krbd.cc b/src/krbd.cc
index 2a1b2c98c0d46e7b6137911d43e976862097f6ad..5b379e6df14b1b3a49f56dc8ff46680d7ba51567 100644 (file)
--- a/src/krbd.cc
+++ b/src/krbd.cc
@@ -42,6 +42,8 @@
 #include "mon/MonMap.h"
 
 #include <blkid/blkid.h>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/tokenizer.hpp>
 #include <libudev.h>
 
 static const int UDEV_BUF_SIZE = 1 << 20;  /* doubled to 2M (SO_RCVBUFFORCE) */
@@ -159,24 +161,39 @@ static int have_minor_attr(void)
 }
 
 static int build_map_buf(CephContext *cct, const krbd_spec& spec,
-                         const char *options, string *pbuf)
+                         const string& options, string *pbuf)
 {
+  bool msgr2 = false;
   ostringstream oss;
   int r;
 
+  boost::char_separator<char> sep(",");
+  boost::tokenizer<boost::char_separator<char>> tok(options, sep);
+  for (const auto& t : tok) {
+    if (boost::starts_with(t, "ms_mode=")) {
+      /* msgr2 unless ms_mode=legacy */
+      msgr2 = t.compare(8, t.npos, "legacy");
+    }
+  }
+
   MonMap monmap;
   r = monmap.build_initial(cct, false, cerr);
   if (r < 0)
     return r;
 
-  list<entity_addr_t> mon_addr;
-  monmap.list_addrs(mon_addr);
-
-  for (const auto &p : mon_addr) {
-    if (oss.tellp() > 0) {
-      oss << ",";
+  /*
+   * If msgr2, filter TYPE_MSGR2 addresses.  Otherwise, filter
+   * TYPE_LEGACY addresses.
+   */
+  for (const auto& p : monmap.mon_info) {
+    for (const auto& a : p.second.public_addrs.v) {
+      if ((msgr2 && a.is_msgr2()) || (!msgr2 && a.is_legacy())) {
+        if (oss.tellp() > 0) {
+          oss << ",";
+        }
+        oss << a.get_sockaddr();
+      }
     }
-    oss << p.get_sockaddr();
   }
 
   oss << " name=" << cct->_conf->name.get_id();
@@ -219,7 +236,7 @@ static int build_map_buf(CephContext *cct, const krbd_spec& spec,
     oss << ",key=" << key_name;
   }
 
-  if (strcmp(options, "") != 0)
+  if (!options.empty())
     oss << "," << options;
   if (!spec.nspace_name.empty())
     oss << ",_pool_ns=" << spec.nspace_name;

diff --git a/src/tools/rbd/action/Kernel.cc b/src/tools/rbd/action/Kernel.cc
index 91eb184bef7ff3b113a37c86fca7a5dd639022bf..dc0938eb0bab74f71f0472521bf9af3a2a24cf78 100644 (file)
--- a/src/tools/rbd/action/Kernel.cc
+++ b/src/tools/rbd/action/Kernel.cc
@@ -64,6 +64,16 @@ static std::string map_option_int_cb(const char *value_char)
   return stringify(d);
 }
 
+static std::string map_option_ms_mode_cb(const char *value_char)
+{
+  if (!strcmp(value_char, "legacy") || !strcmp(value_char, "crc") ||
+      !strcmp(value_char, "secure") || !strcmp(value_char, "prefer-crc") ||
+      !strcmp(value_char, "prefer-secure")) {
+    return value_char;
+  }
+  return "";
+}
+
 static void put_map_option(const std::string &key, const std::string &val)
 {
   map_options[key] = val;
@@ -153,6 +163,9 @@ static int parse_map_options(const std::string &options_string)
     } else if (!strcmp(this_char, "alloc_size")) {
       if (put_map_option_value("alloc_size", value_char, map_option_int_cb))
         return -EINVAL;
+    } else if (!strcmp(this_char, "ms_mode")) {
+      if (put_map_option_value("ms_mode", value_char, map_option_ms_mode_cb))
+        return -EINVAL;
     } else if (!strcmp(this_char, "udev") || !strcmp(this_char, "noudev")) {
       put_map_option("udev", this_char);
     } else {