From 756075915749c318c2b26d29f0164781477a5b2b Mon Sep 17 00:00:00 2001 From: Ramana Raja Date: Mon, 30 Jan 2023 02:21:54 -0500 Subject: [PATCH] mgr: store names of modules that register RADOS clients in the MgrMap The MgrMap stores a list of RADOS clients' addresses registered by the mgr modules. During failover of ceph-mgr, the list is used to blocklist clients belonging to the failed ceph-mgr. Store the names of the mgr modules that registered the RADOS clients along with the clients' addresses in the MgrMap. During debugging, this allows easy identification of the mgr module that registered a particular RADOS client by just dumping the MgrMap (`ceph mgr dump`). Following is the MgrMap output with a module's client name displayed along with its client addrvec, $ ceph mgr dump | jq '.active_clients[0]' { "name": "devicehealth", "addrvec": [ { "type": "v2", "addr": "10.0.0.148:0", "nonce": 612376578 } ] } Fixes: https://tracker.ceph.com/issues/58691 Signed-off-by: Ramana Raja (cherry picked from commit b545fb9f5660dca3af4dea195ea4555f09b3a6e8) --- qa/workunits/cephtool/test.sh | 1 + src/messages/MMgrBeacon.h | 41 ++++++++++++++++++++++++++----- src/mgr/PyModuleRegistry.h | 7 +----- src/mon/MgrMap.h | 45 +++++++++++++++++++++++++++++------ src/mon/MgrMonitor.cc | 2 +- 5 files changed, 76 insertions(+), 20 deletions(-) diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index 7ee0c1d904599..ace8bfe6d5c1a 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -758,6 +758,7 @@ function test_mon_misc() ceph mgr stat ceph mgr dump + ceph mgr dump | jq -e '.active_clients[0].name' ceph mgr module ls ceph mgr module enable restful expect_false ceph mgr module enable foodne diff --git a/src/messages/MMgrBeacon.h b/src/messages/MMgrBeacon.h index 371f296829c6f..1799e2f7168c4 100644 --- a/src/messages/MMgrBeacon.h +++ b/src/messages/MMgrBeacon.h @@ -24,7 +24,7 @@ class MMgrBeacon final : public PaxosServiceMessage { private: - static constexpr int HEAD_VERSION = 10; + static constexpr int HEAD_VERSION = 11; static constexpr int COMPAT_VERSION = 8; protected: @@ -45,7 +45,7 @@ protected: std::map metadata; ///< misc metadata about this osd - std::vector clients; + std::multimap clients; uint64_t mgr_features = 0; ///< reporting mgr's features @@ -59,12 +59,12 @@ public: entity_addrvec_t server_addrs_, bool available_, std::vector&& modules_, std::map&& metadata_, - std::vector clients, + std::multimap&& clients_, uint64_t feat) : PaxosServiceMessage{MSG_MGR_BEACON, 0, HEAD_VERSION, COMPAT_VERSION}, gid(gid_), server_addrs(server_addrs_), available(available_), name(name_), fsid(fsid_), modules(std::move(modules_)), metadata(std::move(metadata_)), - clients(std::move(clients)), + clients(std::move(clients_)), mgr_features(feat) { } @@ -147,7 +147,17 @@ public: encode(modules, payload); encode(mgr_features, payload); - encode(clients, payload, features); + + std::vector clients_names; + std::vector clients_addrs; + for (const auto& i : clients) { + clients_names.push_back(i.first); + clients_addrs.push_back(i.second); + } + // The address vector needs to be encoded first to produce a + // backwards compatible messsage for older monitors. + encode(clients_addrs, payload, features); + encode(clients_names, payload, features); } void decode_payload() override { using ceph::decode; @@ -169,7 +179,26 @@ public: decode(mgr_features, p); } if (header.version >= 10) { - decode(clients, p); + std::vector clients_addrs; + decode(clients_addrs, p); + clients.clear(); + if (header.version >= 11) { + std::vector clients_names; + decode(clients_names, p); + if (clients_names.size() != clients_addrs.size()) { + throw ceph::buffer::malformed_input( + "clients_names.size() != clients_addrs.size()"); + } + auto cn = clients_names.begin(); + auto ca = clients_addrs.begin(); + for(; cn != clients_names.end(); ++cn, ++ca) { + clients.emplace(*cn, *ca); + } + } else { + for (const auto& i : clients_addrs) { + clients.emplace("", i); + } + } } } private: diff --git a/src/mgr/PyModuleRegistry.h b/src/mgr/PyModuleRegistry.h index 4cd98e8827e48..3231d7c667940 100644 --- a/src/mgr/PyModuleRegistry.h +++ b/src/mgr/PyModuleRegistry.h @@ -219,12 +219,7 @@ public: auto get_clients() const { - std::scoped_lock l(lock); - std::vector v; - for (const auto& p : clients) { - v.push_back(p.second); - } - return v; + return clients; } // <<< (end of ActivePyModules cheeky call-throughs) diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h index c06a691b04add..433fdb4ac1ef6 100644 --- a/src/mon/MgrMap.h +++ b/src/mon/MgrMap.h @@ -238,7 +238,7 @@ public: /// features uint64_t active_mgr_features = 0; - std::vector clients; // for blocklist + std::multimap clients; // for blocklist std::map standbys; @@ -394,7 +394,7 @@ public: ENCODE_FINISH(bl); return; } - ENCODE_START(11, 6, bl); + ENCODE_START(12, 6, bl); encode(epoch, bl); encode(active_addrs, bl, features); encode(active_gid, bl); @@ -408,14 +408,23 @@ public: encode(always_on_modules, bl); encode(active_mgr_features, bl); encode(last_failure_osd_epoch, bl); - encode(clients, bl, features); + std::vector clients_names; + std::vector clients_addrs; + for (const auto& i : clients) { + clients_names.push_back(i.first); + clients_addrs.push_back(i.second); + } + // The address vector needs to be encoded first to produce a + // backwards compatible messsage for older monitors. + encode(clients_addrs, bl, features); + encode(clients_names, bl, features); ENCODE_FINISH(bl); return; } void decode(ceph::buffer::list::const_iterator& p) { - DECODE_START(11, p); + DECODE_START(12, p); decode(epoch, p); decode(active_addrs, p); decode(active_gid, p); @@ -461,7 +470,26 @@ public: decode(last_failure_osd_epoch, p); } if (struct_v >= 11) { - decode(clients, p); + std::vector clients_addrs; + decode(clients_addrs, p); + clients.clear(); + if (struct_v >= 12) { + std::vector clients_names; + decode(clients_names, p); + if (clients_names.size() != clients_addrs.size()) { + throw ceph::buffer::malformed_input( + "clients_names.size() != clients_addrs.size()"); + } + auto cn = clients_names.begin(); + auto ca = clients_addrs.begin(); + for(; cn != clients_names.end(); ++cn, ++ca) { + clients.emplace(*cn, *ca); + } + } else { + for (const auto& i : clients_addrs) { + clients.emplace("", i); + } + } } DECODE_FINISH(p); } @@ -517,8 +545,11 @@ public: f->close_section(); // always_on_modules f->dump_int("last_failure_osd_epoch", last_failure_osd_epoch); f->open_array_section("active_clients"); - for (const auto &c : clients) { - f->dump_object("client", c); + for (const auto& i : clients) { + f->open_object_section("client"); + f->dump_string("name", i.first); + i.second.dump(f); + f->close_section(); } f->close_section(); // active_clients } diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc index 170e2b53b5227..32a320a59a523 100644 --- a/src/mon/MgrMonitor.cc +++ b/src/mon/MgrMonitor.cc @@ -910,7 +910,7 @@ void MgrMonitor::drop_active() /* blocklist RADOS clients in use by the mgr */ for (const auto& a : pending_map.clients) { - mon.osdmon()->blocklist(a, until); + mon.osdmon()->blocklist(a.second, until); } request_proposal(mon.osdmon()); -- 2.39.5