From: John Spray <john.spray@redhat.com>
Date: Thu, 29 Jun 2017 19:11:12 +0000 (-0400)
Subject: mgr: transmit command descriptions to mgr in activating beacon
X-Git-Tag: v12.1.2~196^2~9
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5c3846306b6bf7864327265bbac9607295f46cfc;p=ceph.git

mgr: transmit command descriptions to mgr in activating beacon

The mgr already sends a beacon to the mon immediately
after loading python modules in Mgr::init, to indicate
that it is now available.  Use that beacon to transmit
the command descriptions.

The monitor should handle this beacon by persisting
the command descriptions before persisting the updated
mgrmap that indicates that the mgr is now active.

Signed-off-by: John Spray <john.spray@redhat.com>
---

diff --git a/src/messages/MMgrBeacon.h b/src/messages/MMgrBeacon.h
index ab55a9642a21..c2ccee1fc4d3 100644
--- a/src/messages/MMgrBeacon.h
+++ b/src/messages/MMgrBeacon.h
@@ -16,13 +16,14 @@
 #define CEPH_MMGRBEACON_H
 
 #include "messages/PaxosServiceMessage.h"
+#include "mon/MonCommand.h"
 
 #include "include/types.h"
 
 
 class MMgrBeacon : public PaxosServiceMessage {
 
-  static const int HEAD_VERSION = 3;
+  static const int HEAD_VERSION = 4;
   static const int COMPAT_VERSION = 1;
 
 protected:
@@ -33,6 +34,9 @@ protected:
   uuid_d fsid;
   std::set<std::string> available_modules;
 
+  // Only populated during activation
+  std::vector<MonCommand> command_descs;
+
 public:
   MMgrBeacon()
     : PaxosServiceMessage(MSG_MGR_BEACON, 0, HEAD_VERSION, COMPAT_VERSION),
@@ -56,6 +60,16 @@ public:
   const uuid_d& get_fsid() const { return fsid; }
   std::set<std::string>& get_available_modules() { return available_modules; }
 
+  void set_command_descs(const std::vector<MonCommand> &cmds)
+  {
+    command_descs = cmds;
+  }
+
+  const std::vector<MonCommand> &get_command_descs()
+  {
+    return command_descs;
+  }
+
 private:
   ~MMgrBeacon() override {}
 
@@ -77,6 +91,7 @@ public:
     ::encode(name, payload);
     ::encode(fsid, payload);
     ::encode(available_modules, payload);
+    ::encode(command_descs, payload);
   }
   void decode_payload() override {
     bufferlist::iterator p = payload.begin();
@@ -91,6 +106,9 @@ public:
     if (header.version >= 3) {
       ::decode(available_modules, p);
     }
+    if (header.version >= 4) {
+      ::decode(command_descs, p);
+    }
   }
 };
 
diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc
index 3b25ef49010e..e54fd5a62550 100644
--- a/src/mgr/DaemonServer.cc
+++ b/src/mgr/DaemonServer.cc
@@ -33,6 +33,18 @@
 #undef dout_prefix
 #define dout_prefix *_dout << "mgr.server " << __func__ << " "
 
+
+/* The set of statically defined (C++-handled) commands.  This
+ * does not include the Python-defined commands, which are loaded
+ * in PyModules */
+const std::vector<MonCommand> DaemonServer::mgr_commands = {
+#define COMMAND(parsesig, helptext, module, perm, availability) \
+  {parsesig, helptext, module, perm, availability, 0},
+#include "MgrCommands.h"
+#undef COMMAND
+};
+
+
 DaemonServer::DaemonServer(MonClient *monc_,
                            Finisher &finisher_,
 			   DaemonStateIndex &daemon_state_,
@@ -404,12 +416,6 @@ bool DaemonServer::handle_report(MMgrReport *m)
   return true;
 }
 
-std::vector<MonCommand> mgr_commands = {
-#define COMMAND(parsesig, helptext, module, perm, availability) \
-  {parsesig, helptext, module, perm, availability, 0},
-#include "MgrCommands.h"
-#undef COMMAND
-};
 
 void DaemonServer::_generate_command_map(
   map<string,cmd_vartype>& cmdmap,
@@ -582,42 +588,30 @@ bool DaemonServer::handle_command(MCommand *m)
 
   if (prefix == "get_command_descriptions") {
     dout(10) << "reading commands from python modules" << dendl;
-    auto py_commands = py_modules.get_commands();
-
-    bool binary = false;
-    cmd_getval(g_ceph_context, cmdctx->cmdmap, "binary", binary);
-    if (binary) {
-      std::vector<MonCommand> commands = mgr_commands;
-      for (const auto &pyc : py_commands) {
-        commands.push_back({pyc.cmdstring, pyc.helpstring, "mgr",
-                            pyc.perm, "cli", MonCommand::FLAG_MGR});
-      }
-      ::encode(commands, cmdctx->odata);
-    } else {
-      int cmdnum = 0;
-
-      JSONFormatter f;
-      f.open_object_section("command_descriptions");
-      for (const auto &pyc : py_commands) {
-        ostringstream secname;
-        secname << "cmd" << setfill('0') << std::setw(3) << cmdnum;
-        dout(20) << "Dumping " << pyc.cmdstring << " (" << pyc.helpstring
-                 << ")" << dendl;
-        dump_cmddesc_to_json(&f, secname.str(), pyc.cmdstring, pyc.helpstring,
-                             "mgr", pyc.perm, "cli", 0);
-        cmdnum++;
-      }
+    const auto py_commands = py_modules.get_commands();
+
+    int cmdnum = 0;
+    JSONFormatter f;
+    f.open_object_section("command_descriptions");
+
+    auto dump_cmd = [&cmdnum, &f](const MonCommand &mc){
+      ostringstream secname;
+      secname << "cmd" << setfill('0') << std::setw(3) << cmdnum;
+      dump_cmddesc_to_json(&f, secname.str(), mc.cmdstring, mc.helpstring,
+                           mc.module, mc.req_perms, mc.availability, 0);
+      cmdnum++;
+    };
+
+    for (const auto &pyc : py_commands) {
+      dump_cmd(pyc);
+    }
 
-      for (const auto &cp : mgr_commands) {
-        ostringstream secname;
-        secname << "cmd" << setfill('0') << std::setw(3) << cmdnum;
-        dump_cmddesc_to_json(&f, secname.str(), cp.cmdstring, cp.helpstring,
-                             cp.module, cp.req_perms, cp.availability, 0);
-        cmdnum++;
-      }
-      f.close_section();	// command_descriptions
-      f.flush(cmdctx->odata);
+    for (const auto &mgr_cmd : mgr_commands) {
+      dump_cmd(mgr_cmd);
     }
+
+    f.close_section();	// command_descriptions
+    f.flush(cmdctx->odata);
     cmdctx->reply(0, ss);
     return true;
   }
@@ -935,7 +929,7 @@ bool DaemonServer::handle_command(MCommand *m)
 
   // None of the special native commands, 
   MgrPyModule *handler = nullptr;
-  auto py_commands = py_modules.get_commands();
+  auto py_commands = py_modules.get_py_commands();
   for (const auto &pyc : py_commands) {
     auto pyc_prefix = cmddesc_get_prefix(pyc.cmdstring);
     dout(1) << "pyc_prefix: '" << pyc_prefix << "'" << dendl;
diff --git a/src/mgr/DaemonServer.h b/src/mgr/DaemonServer.h
index 6e4483202151..1c5825502a28 100644
--- a/src/mgr/DaemonServer.h
+++ b/src/mgr/DaemonServer.h
@@ -95,6 +95,8 @@ private:
   void maybe_ready(int32_t osd_id);
 
 public:
+  static const std::vector<MonCommand> mgr_commands;
+
   int init(uint64_t gid, entity_addr_t client_addr);
   void shutdown();
 
diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc
index 68126e9430ed..e46b2cabd63b 100644
--- a/src/mgr/Mgr.cc
+++ b/src/mgr/Mgr.cc
@@ -632,3 +632,14 @@ void Mgr::tick()
   dout(10) << dendl;
   server.send_report();
 }
+
+std::vector<MonCommand> Mgr::get_command_set() const
+{
+  Mutex::Locker l(lock);
+
+  std::vector<MonCommand> commands = DaemonServer::mgr_commands;
+  std::vector<MonCommand> py_commands = py_modules.get_commands();
+  commands.insert(commands.end(), py_commands.begin(), py_commands.end());
+  return commands;
+}
+
diff --git a/src/mgr/Mgr.h b/src/mgr/Mgr.h
index ccf22da9e7c4..68f2b40b4616 100644
--- a/src/mgr/Mgr.h
+++ b/src/mgr/Mgr.h
@@ -53,7 +53,7 @@ protected:
   Client    *client;
   Messenger *client_messenger;
 
-  Mutex lock;
+  mutable Mutex lock;
   SafeTimer timer;
   Finisher finisher;
 
@@ -98,6 +98,8 @@ public:
 
   void background_init(Context *completion);
   void shutdown();
+
+  std::vector<MonCommand> get_command_set() const;
 };
 
 #endif
diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc
index 5e82de97f4c9..6f83fc13f326 100644
--- a/src/mgr/MgrStandby.cc
+++ b/src/mgr/MgrStandby.cc
@@ -48,7 +48,8 @@ MgrStandby::MgrStandby(int argc, const char **argv) :
   timer(g_ceph_context, lock),
   active_mgr(nullptr),
   orig_argc(argc),
-  orig_argv(argv)
+  orig_argv(argv),
+  available_in_map(false)
 {
 }
 
@@ -151,7 +152,11 @@ void MgrStandby::send_beacon()
 
   set<string> modules;
   PyModules::list_modules(&modules);
+
+  // Whether I think I am available (request MgrMonitor to set me
+  // as available in the map)
   bool available = active_mgr != nullptr && active_mgr->is_initialized();
+
   auto addr = available ? active_mgr->get_server_addr() : entity_addr_t();
   dout(10) << "sending beacon as gid " << monc.get_global_id()
 	   << " modules " << modules << dendl;
@@ -162,6 +167,16 @@ void MgrStandby::send_beacon()
                                  addr,
                                  available,
 				 modules);
+
+  if (available && !available_in_map) {
+    // We are informing the mon that we are done initializing: inform
+    // it of our command set.  This has to happen after init() because
+    // it needs the python modules to have loaded.
+    m->set_command_descs(active_mgr->get_command_set());
+    dout(4) << "going active, including " << m->get_command_descs().size()
+            << " commands in beacon" << dendl;
+  }
+                                 
   monc.send_mon_message(m);
 }
 
@@ -278,7 +293,7 @@ void MgrStandby::_update_log_config()
 
 void MgrStandby::handle_mgr_map(MMgrMap* mmap)
 {
-  auto map = mmap->get_map();
+  auto &map = mmap->get_map();
   dout(4) << "received map epoch " << map.get_epoch() << dendl;
   const bool active_in_map = map.active_gid == monc.get_global_id();
   dout(4) << "active in map: " << active_in_map
@@ -303,6 +318,11 @@ void MgrStandby::handle_mgr_map(MMgrMap* mmap)
 	respawn();
       }
     }
+
+    if (!available_in_map && map.get_available()) {
+      dout(4) << "Map now says I am available" << dendl;
+      available_in_map = true;
+    }
   } else {
     if (active_mgr != nullptr) {
       derr << "I was active but no longer am" << dendl;
diff --git a/src/mgr/MgrStandby.h b/src/mgr/MgrStandby.h
index 2e2bc6b5f0a2..e24f175cada4 100644
--- a/src/mgr/MgrStandby.h
+++ b/src/mgr/MgrStandby.h
@@ -59,6 +59,8 @@ protected:
   void _update_log_config();
   void send_beacon();
 
+  bool available_in_map;
+
 public:
   MgrStandby(int argc, const char **argv);
   ~MgrStandby() override;
diff --git a/src/mgr/PyModules.cc b/src/mgr/PyModules.cc
index 8412e6cbf5ff..7860a88903f3 100644
--- a/src/mgr/PyModules.cc
+++ b/src/mgr/PyModules.cc
@@ -608,12 +608,12 @@ void PyModules::set_config(const std::string &handle,
   }
 }
 
-std::vector<ModuleCommand> PyModules::get_commands()
+std::vector<ModuleCommand> PyModules::get_py_commands() const
 {
   Mutex::Locker l(lock);
 
   std::vector<ModuleCommand> result;
-  for (auto& i : modules) {
+  for (const auto& i : modules) {
     auto module = i.second.get();
     auto mod_commands = module->get_commands();
     for (auto j : mod_commands) {
@@ -624,6 +624,17 @@ std::vector<ModuleCommand> PyModules::get_commands()
   return result;
 }
 
+std::vector<MonCommand> PyModules::get_commands() const
+{
+  std::vector<ModuleCommand> commands = get_py_commands();
+  std::vector<MonCommand> result;
+  for (auto &pyc: commands) {
+    result.push_back({pyc.cmdstring, pyc.helpstring, "mgr",
+                        pyc.perm, "cli", MonCommand::FLAG_MGR});
+  }
+  return result;
+}
+
 void PyModules::insert_config(const std::map<std::string,
                               std::string> &new_config)
 {
diff --git a/src/mgr/PyModules.h b/src/mgr/PyModules.h
index 467555b51a1f..9d9b8e1ec6ce 100644
--- a/src/mgr/PyModules.h
+++ b/src/mgr/PyModules.h
@@ -24,6 +24,7 @@
 #include "client/Client.h"
 #include "common/LogClient.h"
 #include "mon/MgrMap.h"
+#include "mon/MonCommand.h"
 
 #include "DaemonState.h"
 #include "ClusterState.h"
@@ -80,7 +81,11 @@ public:
 
   std::map<std::string, std::string> config_cache;
 
-  std::vector<ModuleCommand> get_commands();
+  // Python command definitions, including callback
+  std::vector<ModuleCommand> get_py_commands() const;
+
+  // Monitor command definitions, suitable for CLI
+  std::vector<MonCommand> get_commands() const;
 
   void insert_config(const std::map<std::string, std::string> &new_config);