mgr: transmit command descriptions to mgr in activating beacon

The mgr already sends a beacon to the mon immediately
after loading python modules in Mgr::init, to indicate
that it is now available.  Use that beacon to transmit
the command descriptions.

The monitor should handle this beacon by persisting
the command descriptions before persisting the updated
mgrmap that indicates that the mgr is now active.

Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/src/messages/MMgrBeacon.h b/src/messages/MMgrBeacon.h
index ab55a9642a216a1dc8585524f4acc1a42f8179e1..c2ccee1fc4d314fd11ca2a5e686f6010f76511e1 100644 (file)
--- a/src/messages/MMgrBeacon.h
+++ b/src/messages/MMgrBeacon.h
@@ -16,13 +16,14 @@
 #define CEPH_MMGRBEACON_H
 
 #include "messages/PaxosServiceMessage.h"
+#include "mon/MonCommand.h"
 
 #include "include/types.h"
 
 
 class MMgrBeacon : public PaxosServiceMessage {
 
-  static const int HEAD_VERSION = 3;
+  static const int HEAD_VERSION = 4;
   static const int COMPAT_VERSION = 1;
 
 protected:
@@ -33,6 +34,9 @@ protected:
   uuid_d fsid;
   std::set<std::string> available_modules;
 
+  // Only populated during activation
+  std::vector<MonCommand> command_descs;
+
 public:
   MMgrBeacon()
     : PaxosServiceMessage(MSG_MGR_BEACON, 0, HEAD_VERSION, COMPAT_VERSION),
@@ -56,6 +60,16 @@ public:
   const uuid_d& get_fsid() const { return fsid; }
   std::set<std::string>& get_available_modules() { return available_modules; }
 
+  void set_command_descs(const std::vector<MonCommand> &cmds)
+  {
+    command_descs = cmds;
+  }
+
+  const std::vector<MonCommand> &get_command_descs()
+  {
+    return command_descs;
+  }
+
 private:
   ~MMgrBeacon() override {}
 
@@ -77,6 +91,7 @@ public:
     ::encode(name, payload);
     ::encode(fsid, payload);
     ::encode(available_modules, payload);
+    ::encode(command_descs, payload);
   }
   void decode_payload() override {
     bufferlist::iterator p = payload.begin();
@@ -91,6 +106,9 @@ public:
     if (header.version >= 3) {
       ::decode(available_modules, p);
     }
+    if (header.version >= 4) {
+      ::decode(command_descs, p);
+    }
   }
 };
 

diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc
index 3b25ef49010e08daa896cbe94fc1f5f98b95dc04..e54fd5a62550ce7b2b89f1c9b38f36bb11be197d 100644 (file)
--- a/src/mgr/DaemonServer.cc
+++ b/src/mgr/DaemonServer.cc
@@ -33,6 +33,18 @@
 #undef dout_prefix
 #define dout_prefix *_dout << "mgr.server " << __func__ << " "
 
+
+/* The set of statically defined (C++-handled) commands.  This
+ * does not include the Python-defined commands, which are loaded
+ * in PyModules */
+const std::vector<MonCommand> DaemonServer::mgr_commands = {
+#define COMMAND(parsesig, helptext, module, perm, availability) \
+  {parsesig, helptext, module, perm, availability, 0},
+#include "MgrCommands.h"
+#undef COMMAND
+};
+
+
 DaemonServer::DaemonServer(MonClient *monc_,
                            Finisher &finisher_,
                           DaemonStateIndex &daemon_state_,
@@ -404,12 +416,6 @@ bool DaemonServer::handle_report(MMgrReport *m)
   return true;
 }
 
-std::vector<MonCommand> mgr_commands = {
-#define COMMAND(parsesig, helptext, module, perm, availability) \
-  {parsesig, helptext, module, perm, availability, 0},
-#include "MgrCommands.h"
-#undef COMMAND
-};
 
 void DaemonServer::_generate_command_map(
   map<string,cmd_vartype>& cmdmap,
@@ -582,42 +588,30 @@ bool DaemonServer::handle_command(MCommand *m)
 
   if (prefix == "get_command_descriptions") {
     dout(10) << "reading commands from python modules" << dendl;
-    auto py_commands = py_modules.get_commands();
-
-    bool binary = false;
-    cmd_getval(g_ceph_context, cmdctx->cmdmap, "binary", binary);
-    if (binary) {
-      std::vector<MonCommand> commands = mgr_commands;
-      for (const auto &pyc : py_commands) {
-        commands.push_back({pyc.cmdstring, pyc.helpstring, "mgr",
-                            pyc.perm, "cli", MonCommand::FLAG_MGR});
-      }
-      ::encode(commands, cmdctx->odata);
-    } else {
-      int cmdnum = 0;
-
-      JSONFormatter f;
-      f.open_object_section("command_descriptions");
-      for (const auto &pyc : py_commands) {
-        ostringstream secname;
-        secname << "cmd" << setfill('0') << std::setw(3) << cmdnum;
-        dout(20) << "Dumping " << pyc.cmdstring << " (" << pyc.helpstring
-                 << ")" << dendl;
-        dump_cmddesc_to_json(&f, secname.str(), pyc.cmdstring, pyc.helpstring,
-                             "mgr", pyc.perm, "cli", 0);
-        cmdnum++;
-      }
+    const auto py_commands = py_modules.get_commands();
+
+    int cmdnum = 0;
+    JSONFormatter f;
+    f.open_object_section("command_descriptions");
+
+    auto dump_cmd = [&cmdnum, &f](const MonCommand &mc){
+      ostringstream secname;
+      secname << "cmd" << setfill('0') << std::setw(3) << cmdnum;
+      dump_cmddesc_to_json(&f, secname.str(), mc.cmdstring, mc.helpstring,
+                           mc.module, mc.req_perms, mc.availability, 0);
+      cmdnum++;
+    };
+
+    for (const auto &pyc : py_commands) {
+      dump_cmd(pyc);
+    }
 
-      for (const auto &cp : mgr_commands) {
-        ostringstream secname;
-        secname << "cmd" << setfill('0') << std::setw(3) << cmdnum;
-        dump_cmddesc_to_json(&f, secname.str(), cp.cmdstring, cp.helpstring,
-                             cp.module, cp.req_perms, cp.availability, 0);
-        cmdnum++;
-      }
-      f.close_section();       // command_descriptions
-      f.flush(cmdctx->odata);
+    for (const auto &mgr_cmd : mgr_commands) {
+      dump_cmd(mgr_cmd);
     }
+
+    f.close_section(); // command_descriptions
+    f.flush(cmdctx->odata);
     cmdctx->reply(0, ss);
     return true;
   }
@@ -935,7 +929,7 @@ bool DaemonServer::handle_command(MCommand *m)
 
   // None of the special native commands, 
   MgrPyModule *handler = nullptr;
-  auto py_commands = py_modules.get_commands();
+  auto py_commands = py_modules.get_py_commands();
   for (const auto &pyc : py_commands) {
     auto pyc_prefix = cmddesc_get_prefix(pyc.cmdstring);
     dout(1) << "pyc_prefix: '" << pyc_prefix << "'" << dendl;

diff --git a/src/mgr/DaemonServer.h b/src/mgr/DaemonServer.h
index 6e44832021510462107eb2e02b7df18c884c54d1..1c5825502a2819c0a113791598e8eb4afe52954b 100644 (file)
--- a/src/mgr/DaemonServer.h
+++ b/src/mgr/DaemonServer.h
@@ -95,6 +95,8 @@ private:
   void maybe_ready(int32_t osd_id);
 
 public:
+  static const std::vector<MonCommand> mgr_commands;
+
   int init(uint64_t gid, entity_addr_t client_addr);
   void shutdown();
 

diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc
index 68126e9430ed8b37de08cc91c80daec372b3b1e0..e46b2cabd63ba87dbabf952ce725531fa3e34644 100644 (file)
--- a/src/mgr/Mgr.cc
+++ b/src/mgr/Mgr.cc
@@ -632,3 +632,14 @@ void Mgr::tick()
   dout(10) << dendl;
   server.send_report();
 }
+
+std::vector<MonCommand> Mgr::get_command_set() const
+{
+  Mutex::Locker l(lock);
+
+  std::vector<MonCommand> commands = DaemonServer::mgr_commands;
+  std::vector<MonCommand> py_commands = py_modules.get_commands();
+  commands.insert(commands.end(), py_commands.begin(), py_commands.end());
+  return commands;
+}
+

diff --git a/src/mgr/Mgr.h b/src/mgr/Mgr.h
index ccf22da9e7c4e91472f04414880e33efd6060c09..68f2b40b4616d62fdfef22f7f7fad093234586c1 100644 (file)
--- a/src/mgr/Mgr.h
+++ b/src/mgr/Mgr.h
@@ -53,7 +53,7 @@ protected:
   Client    *client;
   Messenger *client_messenger;
 
-  Mutex lock;
+  mutable Mutex lock;
   SafeTimer timer;
   Finisher finisher;
 
@@ -98,6 +98,8 @@ public:
 
   void background_init(Context *completion);
   void shutdown();
+
+  std::vector<MonCommand> get_command_set() const;
 };
 
 #endif

diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc
index 5e82de97f4c96a6360d3d9281cf90b2fd91bda4c..6f83fc13f3267106040d2e04290f822d81911ebf 100644 (file)
--- a/src/mgr/MgrStandby.cc
+++ b/src/mgr/MgrStandby.cc
@@ -48,7 +48,8 @@ MgrStandby::MgrStandby(int argc, const char **argv) :
   timer(g_ceph_context, lock),
   active_mgr(nullptr),
   orig_argc(argc),
-  orig_argv(argv)
+  orig_argv(argv),
+  available_in_map(false)
 {
 }
 
@@ -151,7 +152,11 @@ void MgrStandby::send_beacon()
 
   set<string> modules;
   PyModules::list_modules(&modules);
+
+  // Whether I think I am available (request MgrMonitor to set me
+  // as available in the map)
   bool available = active_mgr != nullptr && active_mgr->is_initialized();
+
   auto addr = available ? active_mgr->get_server_addr() : entity_addr_t();
   dout(10) << "sending beacon as gid " << monc.get_global_id()
           << " modules " << modules << dendl;
@@ -162,6 +167,16 @@ void MgrStandby::send_beacon()
                                  addr,
                                  available,
                                 modules);
+
+  if (available && !available_in_map) {
+    // We are informing the mon that we are done initializing: inform
+    // it of our command set.  This has to happen after init() because
+    // it needs the python modules to have loaded.
+    m->set_command_descs(active_mgr->get_command_set());
+    dout(4) << "going active, including " << m->get_command_descs().size()
+            << " commands in beacon" << dendl;
+  }
+                                 
   monc.send_mon_message(m);
 }
 
@@ -278,7 +293,7 @@ void MgrStandby::_update_log_config()
 
 void MgrStandby::handle_mgr_map(MMgrMap* mmap)
 {
-  auto map = mmap->get_map();
+  auto &map = mmap->get_map();
   dout(4) << "received map epoch " << map.get_epoch() << dendl;
   const bool active_in_map = map.active_gid == monc.get_global_id();
   dout(4) << "active in map: " << active_in_map
@@ -303,6 +318,11 @@ void MgrStandby::handle_mgr_map(MMgrMap* mmap)
        respawn();
       }
     }
+
+    if (!available_in_map && map.get_available()) {
+      dout(4) << "Map now says I am available" << dendl;
+      available_in_map = true;
+    }
   } else {
     if (active_mgr != nullptr) {
       derr << "I was active but no longer am" << dendl;

diff --git a/src/mgr/MgrStandby.h b/src/mgr/MgrStandby.h
index 2e2bc6b5f0a2e368a826d5cf0a107313970482fd..e24f175cada4090845e42157e26d7313f88fa304 100644 (file)
--- a/src/mgr/MgrStandby.h
+++ b/src/mgr/MgrStandby.h
@@ -59,6 +59,8 @@ protected:
   void _update_log_config();
   void send_beacon();
 
+  bool available_in_map;
+
 public:
   MgrStandby(int argc, const char **argv);
   ~MgrStandby() override;

diff --git a/src/mgr/PyModules.cc b/src/mgr/PyModules.cc
index 8412e6cbf5fff238131fcda3065a2d60806f8bc0..7860a88903f34c3f61b8057ab15aba5f1d701ba9 100644 (file)
--- a/src/mgr/PyModules.cc
+++ b/src/mgr/PyModules.cc
@@ -608,12 +608,12 @@ void PyModules::set_config(const std::string &handle,
   }
 }
 
-std::vector<ModuleCommand> PyModules::get_commands()
+std::vector<ModuleCommand> PyModules::get_py_commands() const
 {
   Mutex::Locker l(lock);
 
   std::vector<ModuleCommand> result;
-  for (auto& i : modules) {
+  for (const auto& i : modules) {
     auto module = i.second.get();
     auto mod_commands = module->get_commands();
     for (auto j : mod_commands) {
@@ -624,6 +624,17 @@ std::vector<ModuleCommand> PyModules::get_commands()
   return result;
 }
 
+std::vector<MonCommand> PyModules::get_commands() const
+{
+  std::vector<ModuleCommand> commands = get_py_commands();
+  std::vector<MonCommand> result;
+  for (auto &pyc: commands) {
+    result.push_back({pyc.cmdstring, pyc.helpstring, "mgr",
+                        pyc.perm, "cli", MonCommand::FLAG_MGR});
+  }
+  return result;
+}
+
 void PyModules::insert_config(const std::map<std::string,
                               std::string> &new_config)
 {

diff --git a/src/mgr/PyModules.h b/src/mgr/PyModules.h
index 467555b51a1feca1111631340c1fb08de296e606..9d9b8e1ec6cea863b7a874993e6564d9c5d3a31b 100644 (file)
--- a/src/mgr/PyModules.h
+++ b/src/mgr/PyModules.h
@@ -24,6 +24,7 @@
 #include "client/Client.h"
 #include "common/LogClient.h"
 #include "mon/MgrMap.h"
+#include "mon/MonCommand.h"
 
 #include "DaemonState.h"
 #include "ClusterState.h"
@@ -80,7 +81,11 @@ public:
 
   std::map<std::string, std::string> config_cache;
 
-  std::vector<ModuleCommand> get_commands();
+  // Python command definitions, including callback
+  std::vector<ModuleCommand> get_py_commands() const;
+
+  // Monitor command definitions, suitable for CLI
+  std::vector<MonCommand> get_commands() const;
 
   void insert_config(const std::map<std::string, std::string> &new_config);