From: Ramana Raja Date: Mon, 30 Jan 2023 07:21:54 +0000 (-0500) Subject: mgr: store names of modules that register RADOS clients in the MgrMap X-Git-Tag: v16.2.14~10^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=9baa469a05e31a304903b197646723e80fcf8217;p=ceph.git mgr: store names of modules that register RADOS clients in the MgrMap The MgrMap stores a list of RADOS clients' addresses registered by the mgr modules. During failover of ceph-mgr, the list is used to blocklist clients belonging to the failed ceph-mgr. Store the names of the mgr modules that registered the RADOS clients along with the clients' addresses in the MgrMap. During debugging, this allows easy identification of the mgr module that registered a particular RADOS client by just dumping the MgrMap (`ceph mgr dump`). Following is the MgrMap output with a module's client name displayed along with its client addrvec, $ ceph mgr dump | jq '.active_clients[0]' { "name": "devicehealth", "addrvec": [ { "type": "v2", "addr": "10.0.0.148:0", "nonce": 612376578 } ] } Fixes: https://tracker.ceph.com/issues/58691 Signed-off-by: Ramana Raja (cherry picked from commit b545fb9f5660dca3af4dea195ea4555f09b3a6e8) Conflicts: PendingReleaseNotes [ moved to >=16.2.14 section ] --- diff --git a/PendingReleaseNotes b/PendingReleaseNotes index bc1a59446d73..8322ba3ad3f0 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -38,6 +38,11 @@ * CEPHFS: After recovering a Ceph File System post following the disaster recovery procedure, the recovered files under `lost+found` directory can now be deleted. +* `ceph mgr dump` command now displays the name of the mgr module that + registered a RADOS client in the `name` field added to elements of the + `active_clients` array. Previously, only the address of a module's RADOS + client was shown in the `active_clients` array. + >=16.2.12 --------- @@ -105,10 +110,6 @@ * `ceph mgr dump` command now outputs `last_failure_osd_epoch` and `active_clients` fields at the top level. Previously, these fields were output under `always_on_modules` field. -* `ceph mgr dump` command now displays the name of the mgr module that - registered a RADOS client in the `name` field added to elements of the - `active_clients` array. Previously, only the address of a module's RADOS - client was shown in the `active_clients` array. * RBD: All rbd-mirror daemon perf counters became labeled and as such are now emitted only by the new `counter dump` and `counter schema` commands. As part of the conversion, many also got renamed to better disambiguate journal-based diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index 06e8cbe62bad..2fa2d0d243a1 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -758,6 +758,7 @@ function test_mon_misc() ceph mgr stat ceph mgr dump + ceph mgr dump | jq -e '.active_clients[0].name' ceph mgr module ls ceph mgr module enable restful expect_false ceph mgr module enable foodne diff --git a/src/messages/MMgrBeacon.h b/src/messages/MMgrBeacon.h index 5b6ca3b60be0..d635b9984750 100644 --- a/src/messages/MMgrBeacon.h +++ b/src/messages/MMgrBeacon.h @@ -24,7 +24,7 @@ class MMgrBeacon final : public PaxosServiceMessage { private: - static constexpr int HEAD_VERSION = 10; + static constexpr int HEAD_VERSION = 11; static constexpr int COMPAT_VERSION = 8; protected: @@ -45,7 +45,7 @@ protected: std::map metadata; ///< misc metadata about this osd - std::vector clients; + std::multimap clients; uint64_t mgr_features = 0; ///< reporting mgr's features @@ -59,12 +59,12 @@ public: entity_addrvec_t server_addrs_, bool available_, std::vector&& modules_, std::map&& metadata_, - std::vector clients, + std::multimap&& clients_, uint64_t feat) : PaxosServiceMessage{MSG_MGR_BEACON, 0, HEAD_VERSION, COMPAT_VERSION}, gid(gid_), server_addrs(server_addrs_), available(available_), name(name_), fsid(fsid_), modules(std::move(modules_)), metadata(std::move(metadata_)), - clients(std::move(clients)), + clients(std::move(clients_)), mgr_features(feat) { } @@ -152,7 +152,17 @@ public: encode(modules, payload); encode(mgr_features, payload); - encode(clients, payload, features); + + std::vector clients_names; + std::vector clients_addrs; + for (const auto& i : clients) { + clients_names.push_back(i.first); + clients_addrs.push_back(i.second); + } + // The address vector needs to be encoded first to produce a + // backwards compatible messsage for older monitors. + encode(clients_addrs, payload, features); + encode(clients_names, payload, features); } void decode_payload() override { using ceph::decode; @@ -194,7 +204,26 @@ public: decode(mgr_features, p); } if (header.version >= 10) { - decode(clients, p); + std::vector clients_addrs; + decode(clients_addrs, p); + clients.clear(); + if (header.version >= 11) { + std::vector clients_names; + decode(clients_names, p); + if (clients_names.size() != clients_addrs.size()) { + throw ceph::buffer::malformed_input( + "clients_names.size() != clients_addrs.size()"); + } + auto cn = clients_names.begin(); + auto ca = clients_addrs.begin(); + for(; cn != clients_names.end(); ++cn, ++ca) { + clients.emplace(*cn, *ca); + } + } else { + for (const auto& i : clients_addrs) { + clients.emplace("", i); + } + } } } private: diff --git a/src/mgr/PyModuleRegistry.h b/src/mgr/PyModuleRegistry.h index 6c72af8932a1..17ed8917c78d 100644 --- a/src/mgr/PyModuleRegistry.h +++ b/src/mgr/PyModuleRegistry.h @@ -219,12 +219,7 @@ public: auto get_clients() const { - std::scoped_lock l(lock); - std::vector v; - for (const auto& p : clients) { - v.push_back(p.second); - } - return v; + return clients; } // <<< (end of ActivePyModules cheeky call-throughs) diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h index 6f73d8468d25..f37ed97fd16c 100644 --- a/src/mon/MgrMap.h +++ b/src/mon/MgrMap.h @@ -238,7 +238,7 @@ public: /// features uint64_t active_mgr_features = 0; - std::vector clients; // for blocklist + std::multimap clients; // for blocklist std::map standbys; @@ -401,7 +401,7 @@ public: ENCODE_FINISH(bl); return; } - ENCODE_START(11, 6, bl); + ENCODE_START(12, 6, bl); encode(epoch, bl); encode(active_addrs, bl, features); encode(active_gid, bl); @@ -415,14 +415,23 @@ public: encode(always_on_modules, bl); encode(active_mgr_features, bl); encode(last_failure_osd_epoch, bl); - encode(clients, bl, features); + std::vector clients_names; + std::vector clients_addrs; + for (const auto& i : clients) { + clients_names.push_back(i.first); + clients_addrs.push_back(i.second); + } + // The address vector needs to be encoded first to produce a + // backwards compatible messsage for older monitors. + encode(clients_addrs, bl, features); + encode(clients_names, bl, features); ENCODE_FINISH(bl); return; } void decode(ceph::buffer::list::const_iterator& p) { - DECODE_START(11, p); + DECODE_START(12, p); decode(epoch, p); decode(active_addrs, p); decode(active_gid, p); @@ -468,7 +477,26 @@ public: decode(last_failure_osd_epoch, p); } if (struct_v >= 11) { - decode(clients, p); + std::vector clients_addrs; + decode(clients_addrs, p); + clients.clear(); + if (struct_v >= 12) { + std::vector clients_names; + decode(clients_names, p); + if (clients_names.size() != clients_addrs.size()) { + throw ceph::buffer::malformed_input( + "clients_names.size() != clients_addrs.size()"); + } + auto cn = clients_names.begin(); + auto ca = clients_addrs.begin(); + for(; cn != clients_names.end(); ++cn, ++ca) { + clients.emplace(*cn, *ca); + } + } else { + for (const auto& i : clients_addrs) { + clients.emplace("", i); + } + } } DECODE_FINISH(p); } @@ -524,8 +552,11 @@ public: f->close_section(); // always_on_modules f->dump_int("last_failure_osd_epoch", last_failure_osd_epoch); f->open_array_section("active_clients"); - for (const auto &c : clients) { - f->dump_object("client", c); + for (const auto& i : clients) { + f->open_object_section("client"); + f->dump_string("name", i.first); + i.second.dump(f); + f->close_section(); } f->close_section(); // active_clients } diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc index e4fb76e6fc4a..da869f797941 100644 --- a/src/mon/MgrMonitor.cc +++ b/src/mon/MgrMonitor.cc @@ -919,7 +919,7 @@ bool MgrMonitor::drop_active() /* blocklist RADOS clients in use by the mgr */ for (const auto& a : pending_map.clients) { - mon.osdmon()->blocklist(a, until); + mon.osdmon()->blocklist(a.second, until); } request_proposal(mon.osdmon());