mgr: store names of modules that register RADOS clients in the MgrMap

The MgrMap stores a list of RADOS clients' addresses registered by the
mgr modules. During failover of ceph-mgr, the list is used to blocklist
clients belonging to the failed ceph-mgr.

Store the names of the mgr modules that registered the RADOS clients
along with the clients' addresses in the MgrMap. During debugging, this
allows easy identification of the mgr module that registered a
particular RADOS client by just dumping the MgrMap (`ceph mgr dump`).

Following is the MgrMap output with a module's client name displayed
along with its client addrvec,
$ ceph mgr dump | jq '.active_clients[0]'
{
  "name": "devicehealth",
  "addrvec": [
    {
      "type": "v2",
      "addr": "10.0.0.148:0",
      "nonce": 612376578
    }
  ]
}

Fixes: https://tracker.ceph.com/issues/58691
Signed-off-by: Ramana Raja <rraja@redhat.com>
(cherry picked from commit b545fb9f5660dca3af4dea195ea4555f09b3a6e8)

diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh
index 7ee0c1d9045991656283a604cc8e2a326655bc10..ace8bfe6d5c1a2f4e7ebfe2ac31c288b61df3b56 100755 (executable)
--- a/qa/workunits/cephtool/test.sh
+++ b/qa/workunits/cephtool/test.sh
@@ -758,6 +758,7 @@ function test_mon_misc()
 
   ceph mgr stat
   ceph mgr dump
+  ceph mgr dump | jq -e '.active_clients[0].name'
   ceph mgr module ls
   ceph mgr module enable restful
   expect_false ceph mgr module enable foodne

diff --git a/src/messages/MMgrBeacon.h b/src/messages/MMgrBeacon.h
index 371f296829c6f2ba2c85d60d25d122872d2141f8..1799e2f7168c47691c8af3989a2330f0417d3188 100644 (file)
--- a/src/messages/MMgrBeacon.h
+++ b/src/messages/MMgrBeacon.h
@@ -24,7 +24,7 @@
 
 class MMgrBeacon final : public PaxosServiceMessage {
 private:
-  static constexpr int HEAD_VERSION = 10;
+  static constexpr int HEAD_VERSION = 11;
   static constexpr int COMPAT_VERSION = 8;
 
 protected:
@@ -45,7 +45,7 @@ protected:
 
   std::map<std::string,std::string> metadata; ///< misc metadata about this osd
 
-  std::vector<entity_addrvec_t> clients;
+  std::multimap<std::string, entity_addrvec_t> clients;
 
   uint64_t mgr_features = 0;   ///< reporting mgr's features
 
@@ -59,12 +59,12 @@ public:
              entity_addrvec_t server_addrs_, bool available_,
             std::vector<MgrMap::ModuleInfo>&& modules_,
             std::map<std::string,std::string>&& metadata_,
-             std::vector<entity_addrvec_t> clients,
+             std::multimap<std::string, entity_addrvec_t>&& clients_,
             uint64_t feat)
     : PaxosServiceMessage{MSG_MGR_BEACON, 0, HEAD_VERSION, COMPAT_VERSION},
       gid(gid_), server_addrs(server_addrs_), available(available_), name(name_),
       fsid(fsid_), modules(std::move(modules_)), metadata(std::move(metadata_)),
-      clients(std::move(clients)),
+      clients(std::move(clients_)),
       mgr_features(feat)
   {
   }
@@ -147,7 +147,17 @@ public:
 
     encode(modules, payload);
     encode(mgr_features, payload);
-    encode(clients, payload, features);
+
+    std::vector<std::string> clients_names;
+    std::vector<entity_addrvec_t> clients_addrs;
+    for (const auto& i : clients) {
+      clients_names.push_back(i.first);
+      clients_addrs.push_back(i.second);
+    }
+    // The address vector needs to be encoded first to produce a
+    // backwards compatible messsage for older monitors.
+    encode(clients_addrs, payload, features);
+    encode(clients_names, payload, features);
   }
   void decode_payload() override {
     using ceph::decode;
@@ -169,7 +179,26 @@ public:
       decode(mgr_features, p);
     }
     if (header.version >= 10) {
-      decode(clients, p);
+      std::vector<entity_addrvec_t> clients_addrs;
+      decode(clients_addrs, p);
+      clients.clear();
+      if (header.version >= 11) {
+       std::vector<std::string> clients_names;
+       decode(clients_names, p);
+       if (clients_names.size() != clients_addrs.size()) {
+         throw ceph::buffer::malformed_input(
+           "clients_names.size() != clients_addrs.size()");
+       }
+       auto cn = clients_names.begin();
+       auto ca = clients_addrs.begin();
+       for(; cn != clients_names.end(); ++cn, ++ca) {
+         clients.emplace(*cn, *ca);
+       }
+      } else {
+       for (const auto& i : clients_addrs) {
+         clients.emplace("", i);
+       }
+      }
     }
   }
 private:

diff --git a/src/mgr/PyModuleRegistry.h b/src/mgr/PyModuleRegistry.h
index 4cd98e8827e487f7308146469020733d1e2d2640..3231d7c667940266ecf88a2936f659392bde31e3 100644 (file)
--- a/src/mgr/PyModuleRegistry.h
+++ b/src/mgr/PyModuleRegistry.h
@@ -219,12 +219,7 @@ public:
 
   auto get_clients() const
   {
-    std::scoped_lock l(lock);
-    std::vector<entity_addrvec_t> v;
-    for (const auto& p : clients) {
-      v.push_back(p.second);
-    }
-    return v;
+    return clients;
   }
 
   // <<< (end of ActivePyModules cheeky call-throughs)

diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h
index c06a691b04add583673b3f46f57825734ce02486..433fdb4ac1ef60059d52d890bef02f586526b58d 100644 (file)
--- a/src/mon/MgrMap.h
+++ b/src/mon/MgrMap.h
@@ -238,7 +238,7 @@ public:
   /// features
   uint64_t active_mgr_features = 0;
 
-  std::vector<entity_addrvec_t> clients; // for blocklist
+  std::multimap<std::string, entity_addrvec_t> clients; // for blocklist
 
   std::map<uint64_t, StandbyInfo> standbys;
 
@@ -394,7 +394,7 @@ public:
       ENCODE_FINISH(bl);
       return;
     }
-    ENCODE_START(11, 6, bl);
+    ENCODE_START(12, 6, bl);
     encode(epoch, bl);
     encode(active_addrs, bl, features);
     encode(active_gid, bl);
@@ -408,14 +408,23 @@ public:
     encode(always_on_modules, bl);
     encode(active_mgr_features, bl);
     encode(last_failure_osd_epoch, bl);
-    encode(clients, bl, features);
+    std::vector<std::string> clients_names;
+    std::vector<entity_addrvec_t> clients_addrs;
+    for (const auto& i : clients) {
+      clients_names.push_back(i.first);
+      clients_addrs.push_back(i.second);
+    }
+    // The address vector needs to be encoded first to produce a
+    // backwards compatible messsage for older monitors.
+    encode(clients_addrs, bl, features);
+    encode(clients_names, bl, features);
     ENCODE_FINISH(bl);
     return;
   }
 
   void decode(ceph::buffer::list::const_iterator& p)
   {
-    DECODE_START(11, p);
+    DECODE_START(12, p);
     decode(epoch, p);
     decode(active_addrs, p);
     decode(active_gid, p);
@@ -461,7 +470,26 @@ public:
       decode(last_failure_osd_epoch, p);
     }
     if (struct_v >= 11) {
-      decode(clients, p);
+      std::vector<entity_addrvec_t> clients_addrs;
+      decode(clients_addrs, p);
+      clients.clear();
+      if (struct_v >= 12) {
+       std::vector<std::string> clients_names;
+       decode(clients_names, p);
+       if (clients_names.size() != clients_addrs.size()) {
+         throw ceph::buffer::malformed_input(
+           "clients_names.size() != clients_addrs.size()");
+       }
+       auto cn = clients_names.begin();
+       auto ca = clients_addrs.begin();
+       for(; cn != clients_names.end(); ++cn, ++ca) {
+         clients.emplace(*cn, *ca);
+       }
+      } else {
+       for (const auto& i : clients_addrs) {
+         clients.emplace("", i);
+       }
+      }
     }
     DECODE_FINISH(p);
   }
@@ -517,8 +545,11 @@ public:
     f->close_section(); // always_on_modules
     f->dump_int("last_failure_osd_epoch", last_failure_osd_epoch);
     f->open_array_section("active_clients");
-    for (const auto &c : clients) {
-      f->dump_object("client", c);
+    for (const auto& i : clients) {
+      f->open_object_section("client");
+      f->dump_string("name", i.first);
+      i.second.dump(f);
+      f->close_section();
     }
     f->close_section(); // active_clients
   }

diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc
index 170e2b53b5227bf6482d4f45dcd7e824205471f6..32a320a59a5232e479fd0fa5128b2a897ed7db3a 100644 (file)
--- a/src/mon/MgrMonitor.cc
+++ b/src/mon/MgrMonitor.cc
@@ -910,7 +910,7 @@ void MgrMonitor::drop_active()
 
   /* blocklist RADOS clients in use by the mgr */
   for (const auto& a : pending_map.clients) {
-    mon.osdmon()->blocklist(a, until);
+    mon.osdmon()->blocklist(a.second, until);
   }
   request_proposal(mon.osdmon());