From: Ilya Dryomov Date: Mon, 18 Jan 2021 12:49:49 +0000 (+0100) Subject: krbd: add support for msgr2 X-Git-Tag: v17.1.0~3173^2~2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=08f714964b7fe5024504818f01328a41acc24965;p=ceph-ci.git krbd: add support for msgr2 Recognize ms_mode map option and filter initial monitor addresses accordingly: if ms_mode is not given or ms_mode=legacy, discard v2 addresses, otherwise discard v1 addresses. Note that nothing was discarded (i.e. v2 addresses were passed to the kernel) previously. The intent was to preserve that behaviour in case ms_mode is not given, allowing to change the kernel default in the future. However, it turns out that mount.ceph helper has been misguidedly discarding v2 addresses since commit eae01275134e ("mount.ceph: fork a child to get info from local configuration"), so that ship has sailed. Fixes: https://tracker.ceph.com/issues/48976 Signed-off-by: Ilya Dryomov --- diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst index 94cdd0bfdde..92307e24ac0 100644 --- a/doc/man/8/rbd.rst +++ b/doc/man/8/rbd.rst @@ -744,9 +744,13 @@ Per client instance `rbd device map` options: * noshare - Disable sharing of client instances with other mappings. -* crc - Enable CRC32C checksumming for data writes (default). +* crc - Enable CRC32C checksumming for msgr1 on-the-wire protocol (default). + For msgr2.1 protocol this option is ignored: full checksumming is always on + in 'crc' mode and always off in 'secure' mode. -* nocrc - Disable CRC32C checksumming for data writes. +* nocrc - Disable CRC32C checksumming for msgr1 on-the-wire protocol. Note + that only payload checksumming is disabled, header checksumming is always on. + For msgr2.1 protocol this option is ignored. * cephx_require_signatures - Require cephx message signing (since 3.19, default). @@ -759,9 +763,12 @@ Per client instance `rbd device map` options: * notcp_nodelay - Enable Nagle's algorithm on client sockets (since 4.0). -* cephx_sign_messages - Enable message signing (since 4.4, default). +* cephx_sign_messages - Enable message signing for msgr1 on-the-wire protocol + (since 4.4, default). For msgr2.1 protocol this option is ignored: message + signing is built into 'secure' mode and not offered in 'crc' mode. -* nocephx_sign_messages - Disable message signing (since 4.4). +* nocephx_sign_messages - Disable message signing for msgr1 on-the-wire protocol + (since 4.4). For msgr2.1 protocol this option is ignored. * mount_timeout=x - A timeout on various steps in `rbd device map` and `rbd device unmap` sequences (default is 60 seconds). In particular, @@ -856,6 +863,25 @@ Per mapping (block device) `rbd device map` options: backend that the data is incompressible, disabling compression in aggressive mode (since 5.8). +* ms_mode=legacy - Use msgr1 on-the-wire protocol (since 5.11, default). + +* ms_mode=crc - Use msgr2.1 on-the-wire protocol, select 'crc' mode, also + referred to as plain mode (since 5.11). If the daemon denies 'crc' mode, + fail the connection. + +* ms_mode=secure - Use msgr2.1 on-the-wire protocol, select 'secure' mode + (since 5.11). 'secure' mode provides full in-transit encryption ensuring + both confidentiality and authenticity. If the daemon denies 'secure' mode, + fail the connection. + +* ms_mode=prefer-crc - Use msgr2.1 on-the-wire protocol, select 'crc' + mode (since 5.11). If the daemon denies 'crc' mode in favor of 'secure' + mode, agree to 'secure' mode. + +* ms_mode=prefer-secure - Use msgr2.1 on-the-wire protocol, select 'secure' + mode (since 5.11). If the daemon denies 'secure' mode in favor of 'crc' + mode, agree to 'crc' mode. + * udev - Wait for udev device manager to finish executing all matching "add" rules and release the device before exiting (default). This option is not passed to the kernel. diff --git a/src/krbd.cc b/src/krbd.cc index 0d463dea52e..5e011d7a9a0 100644 --- a/src/krbd.cc +++ b/src/krbd.cc @@ -44,6 +44,7 @@ #include #include +#include #include static const int UDEV_BUF_SIZE = 1 << 20; /* doubled to 2M (SO_RCVBUFFORCE) */ @@ -184,24 +185,39 @@ static int have_minor_attr(void) } static int build_map_buf(CephContext *cct, const krbd_spec& spec, - const char *options, string *pbuf) + const string& options, string *pbuf) { + bool msgr2 = false; ostringstream oss; int r; + boost::char_separator sep(","); + boost::tokenizer> tok(options, sep); + for (const auto& t : tok) { + if (boost::starts_with(t, "ms_mode=")) { + /* msgr2 unless ms_mode=legacy */ + msgr2 = t.compare(8, t.npos, "legacy"); + } + } + MonMap monmap; r = monmap.build_initial(cct, false, cerr); if (r < 0) return r; - list mon_addr; - monmap.list_addrs(mon_addr); - - for (const auto &p : mon_addr) { - if (oss.tellp() > 0) { - oss << ","; + /* + * If msgr2, filter TYPE_MSGR2 addresses. Otherwise, filter + * TYPE_LEGACY addresses. + */ + for (const auto& p : monmap.mon_info) { + for (const auto& a : p.second.public_addrs.v) { + if ((msgr2 && a.is_msgr2()) || (!msgr2 && a.is_legacy())) { + if (oss.tellp() > 0) { + oss << ","; + } + oss << a.get_sockaddr(); + } } - oss << p.get_sockaddr(); } oss << " name=" << cct->_conf->name.get_id(); @@ -244,7 +260,7 @@ static int build_map_buf(CephContext *cct, const krbd_spec& spec, oss << ",key=" << key_name; } - if (strcmp(options, "") != 0) + if (!options.empty()) oss << "," << options; if (!spec.nspace_name.empty()) oss << ",_pool_ns=" << spec.nspace_name; diff --git a/src/tools/rbd/action/Kernel.cc b/src/tools/rbd/action/Kernel.cc index deddcea3d01..3ed8e59037b 100644 --- a/src/tools/rbd/action/Kernel.cc +++ b/src/tools/rbd/action/Kernel.cc @@ -83,6 +83,16 @@ static std::string map_option_compression_hint_cb(const char *value_char) return ""; } +static std::string map_option_ms_mode_cb(const char *value_char) +{ + if (!strcmp(value_char, "legacy") || !strcmp(value_char, "crc") || + !strcmp(value_char, "secure") || !strcmp(value_char, "prefer-crc") || + !strcmp(value_char, "prefer-secure")) { + return value_char; + } + return ""; +} + static void put_map_option(const std::string &key, const std::string &val, MapOptions* map_options) { @@ -196,6 +206,10 @@ static int parse_map_options(const std::string &options_string, if (put_map_option_value("compression_hint", value_char, map_option_compression_hint_cb, map_options)) return -EINVAL; + } else if (!strcmp(this_char, "ms_mode")) { + if (put_map_option_value("ms_mode", value_char, map_option_ms_mode_cb, + map_options)) + return -EINVAL; } else if (!strcmp(this_char, "udev") || !strcmp(this_char, "noudev")) { put_map_option("udev", this_char, map_options); } else {