From 5c3846306b6bf7864327265bbac9607295f46cfc Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 29 Jun 2017 15:11:12 -0400 Subject: [PATCH] mgr: transmit command descriptions to mgr in activating beacon The mgr already sends a beacon to the mon immediately after loading python modules in Mgr::init, to indicate that it is now available. Use that beacon to transmit the command descriptions. The monitor should handle this beacon by persisting the command descriptions before persisting the updated mgrmap that indicates that the mgr is now active. Signed-off-by: John Spray --- src/messages/MMgrBeacon.h | 20 ++++++++++- src/mgr/DaemonServer.cc | 76 ++++++++++++++++++--------------------- src/mgr/DaemonServer.h | 2 ++ src/mgr/Mgr.cc | 11 ++++++ src/mgr/Mgr.h | 4 ++- src/mgr/MgrStandby.cc | 24 +++++++++++-- src/mgr/MgrStandby.h | 2 ++ src/mgr/PyModules.cc | 15 ++++++-- src/mgr/PyModules.h | 7 +++- 9 files changed, 113 insertions(+), 48 deletions(-) diff --git a/src/messages/MMgrBeacon.h b/src/messages/MMgrBeacon.h index ab55a9642a2..c2ccee1fc4d 100644 --- a/src/messages/MMgrBeacon.h +++ b/src/messages/MMgrBeacon.h @@ -16,13 +16,14 @@ #define CEPH_MMGRBEACON_H #include "messages/PaxosServiceMessage.h" +#include "mon/MonCommand.h" #include "include/types.h" class MMgrBeacon : public PaxosServiceMessage { - static const int HEAD_VERSION = 3; + static const int HEAD_VERSION = 4; static const int COMPAT_VERSION = 1; protected: @@ -33,6 +34,9 @@ protected: uuid_d fsid; std::set available_modules; + // Only populated during activation + std::vector command_descs; + public: MMgrBeacon() : PaxosServiceMessage(MSG_MGR_BEACON, 0, HEAD_VERSION, COMPAT_VERSION), @@ -56,6 +60,16 @@ public: const uuid_d& get_fsid() const { return fsid; } std::set& get_available_modules() { return available_modules; } + void set_command_descs(const std::vector &cmds) + { + command_descs = cmds; + } + + const std::vector &get_command_descs() + { + return command_descs; + } + private: ~MMgrBeacon() override {} @@ -77,6 +91,7 @@ public: ::encode(name, payload); ::encode(fsid, payload); ::encode(available_modules, payload); + ::encode(command_descs, payload); } void decode_payload() override { bufferlist::iterator p = payload.begin(); @@ -91,6 +106,9 @@ public: if (header.version >= 3) { ::decode(available_modules, p); } + if (header.version >= 4) { + ::decode(command_descs, p); + } } }; diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc index 3b25ef49010..e54fd5a6255 100644 --- a/src/mgr/DaemonServer.cc +++ b/src/mgr/DaemonServer.cc @@ -33,6 +33,18 @@ #undef dout_prefix #define dout_prefix *_dout << "mgr.server " << __func__ << " " + +/* The set of statically defined (C++-handled) commands. This + * does not include the Python-defined commands, which are loaded + * in PyModules */ +const std::vector DaemonServer::mgr_commands = { +#define COMMAND(parsesig, helptext, module, perm, availability) \ + {parsesig, helptext, module, perm, availability, 0}, +#include "MgrCommands.h" +#undef COMMAND +}; + + DaemonServer::DaemonServer(MonClient *monc_, Finisher &finisher_, DaemonStateIndex &daemon_state_, @@ -404,12 +416,6 @@ bool DaemonServer::handle_report(MMgrReport *m) return true; } -std::vector mgr_commands = { -#define COMMAND(parsesig, helptext, module, perm, availability) \ - {parsesig, helptext, module, perm, availability, 0}, -#include "MgrCommands.h" -#undef COMMAND -}; void DaemonServer::_generate_command_map( map& cmdmap, @@ -582,42 +588,30 @@ bool DaemonServer::handle_command(MCommand *m) if (prefix == "get_command_descriptions") { dout(10) << "reading commands from python modules" << dendl; - auto py_commands = py_modules.get_commands(); - - bool binary = false; - cmd_getval(g_ceph_context, cmdctx->cmdmap, "binary", binary); - if (binary) { - std::vector commands = mgr_commands; - for (const auto &pyc : py_commands) { - commands.push_back({pyc.cmdstring, pyc.helpstring, "mgr", - pyc.perm, "cli", MonCommand::FLAG_MGR}); - } - ::encode(commands, cmdctx->odata); - } else { - int cmdnum = 0; - - JSONFormatter f; - f.open_object_section("command_descriptions"); - for (const auto &pyc : py_commands) { - ostringstream secname; - secname << "cmd" << setfill('0') << std::setw(3) << cmdnum; - dout(20) << "Dumping " << pyc.cmdstring << " (" << pyc.helpstring - << ")" << dendl; - dump_cmddesc_to_json(&f, secname.str(), pyc.cmdstring, pyc.helpstring, - "mgr", pyc.perm, "cli", 0); - cmdnum++; - } + const auto py_commands = py_modules.get_commands(); + + int cmdnum = 0; + JSONFormatter f; + f.open_object_section("command_descriptions"); + + auto dump_cmd = [&cmdnum, &f](const MonCommand &mc){ + ostringstream secname; + secname << "cmd" << setfill('0') << std::setw(3) << cmdnum; + dump_cmddesc_to_json(&f, secname.str(), mc.cmdstring, mc.helpstring, + mc.module, mc.req_perms, mc.availability, 0); + cmdnum++; + }; + + for (const auto &pyc : py_commands) { + dump_cmd(pyc); + } - for (const auto &cp : mgr_commands) { - ostringstream secname; - secname << "cmd" << setfill('0') << std::setw(3) << cmdnum; - dump_cmddesc_to_json(&f, secname.str(), cp.cmdstring, cp.helpstring, - cp.module, cp.req_perms, cp.availability, 0); - cmdnum++; - } - f.close_section(); // command_descriptions - f.flush(cmdctx->odata); + for (const auto &mgr_cmd : mgr_commands) { + dump_cmd(mgr_cmd); } + + f.close_section(); // command_descriptions + f.flush(cmdctx->odata); cmdctx->reply(0, ss); return true; } @@ -935,7 +929,7 @@ bool DaemonServer::handle_command(MCommand *m) // None of the special native commands, MgrPyModule *handler = nullptr; - auto py_commands = py_modules.get_commands(); + auto py_commands = py_modules.get_py_commands(); for (const auto &pyc : py_commands) { auto pyc_prefix = cmddesc_get_prefix(pyc.cmdstring); dout(1) << "pyc_prefix: '" << pyc_prefix << "'" << dendl; diff --git a/src/mgr/DaemonServer.h b/src/mgr/DaemonServer.h index 6e448320215..1c5825502a2 100644 --- a/src/mgr/DaemonServer.h +++ b/src/mgr/DaemonServer.h @@ -95,6 +95,8 @@ private: void maybe_ready(int32_t osd_id); public: + static const std::vector mgr_commands; + int init(uint64_t gid, entity_addr_t client_addr); void shutdown(); diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc index 68126e9430e..e46b2cabd63 100644 --- a/src/mgr/Mgr.cc +++ b/src/mgr/Mgr.cc @@ -632,3 +632,14 @@ void Mgr::tick() dout(10) << dendl; server.send_report(); } + +std::vector Mgr::get_command_set() const +{ + Mutex::Locker l(lock); + + std::vector commands = DaemonServer::mgr_commands; + std::vector py_commands = py_modules.get_commands(); + commands.insert(commands.end(), py_commands.begin(), py_commands.end()); + return commands; +} + diff --git a/src/mgr/Mgr.h b/src/mgr/Mgr.h index ccf22da9e7c..68f2b40b461 100644 --- a/src/mgr/Mgr.h +++ b/src/mgr/Mgr.h @@ -53,7 +53,7 @@ protected: Client *client; Messenger *client_messenger; - Mutex lock; + mutable Mutex lock; SafeTimer timer; Finisher finisher; @@ -98,6 +98,8 @@ public: void background_init(Context *completion); void shutdown(); + + std::vector get_command_set() const; }; #endif diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc index 5e82de97f4c..6f83fc13f32 100644 --- a/src/mgr/MgrStandby.cc +++ b/src/mgr/MgrStandby.cc @@ -48,7 +48,8 @@ MgrStandby::MgrStandby(int argc, const char **argv) : timer(g_ceph_context, lock), active_mgr(nullptr), orig_argc(argc), - orig_argv(argv) + orig_argv(argv), + available_in_map(false) { } @@ -151,7 +152,11 @@ void MgrStandby::send_beacon() set modules; PyModules::list_modules(&modules); + + // Whether I think I am available (request MgrMonitor to set me + // as available in the map) bool available = active_mgr != nullptr && active_mgr->is_initialized(); + auto addr = available ? active_mgr->get_server_addr() : entity_addr_t(); dout(10) << "sending beacon as gid " << monc.get_global_id() << " modules " << modules << dendl; @@ -162,6 +167,16 @@ void MgrStandby::send_beacon() addr, available, modules); + + if (available && !available_in_map) { + // We are informing the mon that we are done initializing: inform + // it of our command set. This has to happen after init() because + // it needs the python modules to have loaded. + m->set_command_descs(active_mgr->get_command_set()); + dout(4) << "going active, including " << m->get_command_descs().size() + << " commands in beacon" << dendl; + } + monc.send_mon_message(m); } @@ -278,7 +293,7 @@ void MgrStandby::_update_log_config() void MgrStandby::handle_mgr_map(MMgrMap* mmap) { - auto map = mmap->get_map(); + auto &map = mmap->get_map(); dout(4) << "received map epoch " << map.get_epoch() << dendl; const bool active_in_map = map.active_gid == monc.get_global_id(); dout(4) << "active in map: " << active_in_map @@ -303,6 +318,11 @@ void MgrStandby::handle_mgr_map(MMgrMap* mmap) respawn(); } } + + if (!available_in_map && map.get_available()) { + dout(4) << "Map now says I am available" << dendl; + available_in_map = true; + } } else { if (active_mgr != nullptr) { derr << "I was active but no longer am" << dendl; diff --git a/src/mgr/MgrStandby.h b/src/mgr/MgrStandby.h index 2e2bc6b5f0a..e24f175cada 100644 --- a/src/mgr/MgrStandby.h +++ b/src/mgr/MgrStandby.h @@ -59,6 +59,8 @@ protected: void _update_log_config(); void send_beacon(); + bool available_in_map; + public: MgrStandby(int argc, const char **argv); ~MgrStandby() override; diff --git a/src/mgr/PyModules.cc b/src/mgr/PyModules.cc index 8412e6cbf5f..7860a88903f 100644 --- a/src/mgr/PyModules.cc +++ b/src/mgr/PyModules.cc @@ -608,12 +608,12 @@ void PyModules::set_config(const std::string &handle, } } -std::vector PyModules::get_commands() +std::vector PyModules::get_py_commands() const { Mutex::Locker l(lock); std::vector result; - for (auto& i : modules) { + for (const auto& i : modules) { auto module = i.second.get(); auto mod_commands = module->get_commands(); for (auto j : mod_commands) { @@ -624,6 +624,17 @@ std::vector PyModules::get_commands() return result; } +std::vector PyModules::get_commands() const +{ + std::vector commands = get_py_commands(); + std::vector result; + for (auto &pyc: commands) { + result.push_back({pyc.cmdstring, pyc.helpstring, "mgr", + pyc.perm, "cli", MonCommand::FLAG_MGR}); + } + return result; +} + void PyModules::insert_config(const std::map &new_config) { diff --git a/src/mgr/PyModules.h b/src/mgr/PyModules.h index 467555b51a1..9d9b8e1ec6c 100644 --- a/src/mgr/PyModules.h +++ b/src/mgr/PyModules.h @@ -24,6 +24,7 @@ #include "client/Client.h" #include "common/LogClient.h" #include "mon/MgrMap.h" +#include "mon/MonCommand.h" #include "DaemonState.h" #include "ClusterState.h" @@ -80,7 +81,11 @@ public: std::map config_cache; - std::vector get_commands(); + // Python command definitions, including callback + std::vector get_py_commands() const; + + // Monitor command definitions, suitable for CLI + std::vector get_commands() const; void insert_config(const std::map &new_config); -- 2.39.5