From d953198aff3e5aaa6e2eadcf8a53c9e0279a30de Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 27 Jun 2017 18:12:35 -0400 Subject: [PATCH] mon,mgr: track available modules in MgrMap too Require --force to enable a module that isn't reported as available. Signed-off-by: Sage Weil --- qa/workunits/cephtool/test.sh | 7 +++++-- src/messages/MMgrBeacon.h | 16 ++++++++++---- src/mgr/MgrStandby.cc | 10 ++++++--- src/mgr/PyModules.cc | 29 ++++++++++++++++++++++++++ src/mgr/PyModules.h | 2 ++ src/mon/MgrMap.h | 39 +++++++++++++++++++++++++++++++---- src/mon/MgrMonitor.cc | 29 +++++++++++++++++++++++++- src/mon/MonCommands.h | 9 ++++---- 8 files changed, 123 insertions(+), 18 deletions(-) diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index 0f6b428c3cd..bbe314b7b14 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -713,8 +713,11 @@ function test_mon_misc() ceph mgr dump ceph mgr module ls - ceph mgr module enable foo - ceph mgr module disable foo + ceph mgr module enable restful + expect_false ceph mgr module enable foodne + ceph mgr module enable foodne --force + ceph mgr module disable foodne + ceph mgr module disable foodnebizbangbash ceph mon metadata a ceph mon metadata diff --git a/src/messages/MMgrBeacon.h b/src/messages/MMgrBeacon.h index 2962d9c9a1f..ab55a9642a2 100644 --- a/src/messages/MMgrBeacon.h +++ b/src/messages/MMgrBeacon.h @@ -22,7 +22,7 @@ class MMgrBeacon : public PaxosServiceMessage { - static const int HEAD_VERSION = 2; + static const int HEAD_VERSION = 3; static const int COMPAT_VERSION = 1; protected: @@ -31,6 +31,7 @@ protected: bool available; std::string name; uuid_d fsid; + std::set available_modules; public: MMgrBeacon() @@ -40,10 +41,11 @@ public: } MMgrBeacon(const uuid_d& fsid_, uint64_t gid_, const std::string &name_, - entity_addr_t server_addr_, bool available_) + entity_addr_t server_addr_, bool available_, + const std::set& module_list) : PaxosServiceMessage(MSG_MGR_BEACON, 0, HEAD_VERSION, COMPAT_VERSION), gid(gid_), server_addr(server_addr_), available(available_), name(name_), - fsid(fsid_) + fsid(fsid_), available_modules(module_list) { } @@ -52,6 +54,7 @@ public: bool get_available() const { return available; } const std::string& get_name() const { return name; } const uuid_d& get_fsid() const { return fsid; } + std::set& get_available_modules() { return available_modules; } private: ~MMgrBeacon() override {} @@ -62,7 +65,8 @@ public: void print(ostream& out) const override { out << get_type_name() << " mgr." << name << "(" << fsid << "," - << gid << ", " << server_addr << ", " << available << ")"; + << gid << ", " << server_addr << ", " << available + << ")"; } void encode_payload(uint64_t features) override { @@ -72,6 +76,7 @@ public: ::encode(available, payload); ::encode(name, payload); ::encode(fsid, payload); + ::encode(available_modules, payload); } void decode_payload() override { bufferlist::iterator p = payload.begin(); @@ -83,6 +88,9 @@ public: if (header.version >= 2) { ::decode(fsid, p); } + if (header.version >= 3) { + ::decode(available_modules, p); + } } }; diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc index bc7422ba630..a3c3234a208 100644 --- a/src/mgr/MgrStandby.cc +++ b/src/mgr/MgrStandby.cc @@ -148,16 +148,20 @@ void MgrStandby::send_beacon() { assert(lock.is_locked_by_me()); dout(1) << state_str() << dendl; - dout(10) << "sending beacon as gid " << monc.get_global_id() << dendl; + set modules; + PyModules::list_modules(&modules); bool available = active_mgr != nullptr && active_mgr->is_initialized(); auto addr = available ? active_mgr->get_server_addr() : entity_addr_t(); + dout(10) << "sending beacon as gid " << monc.get_global_id() + << " modules " << modules << dendl; + MMgrBeacon *m = new MMgrBeacon(monc.get_fsid(), monc.get_global_id(), g_conf->name.get_id(), addr, - available); - + available, + modules); monc.send_mon_message(m); } diff --git a/src/mgr/PyModules.cc b/src/mgr/PyModules.cc index 7d091a253e1..96b2698e3c3 100644 --- a/src/mgr/PyModules.cc +++ b/src/mgr/PyModules.cc @@ -674,3 +674,32 @@ PyObject *PyModules::get_context() return capsule; } +static void _list_modules( + const std::string path, + std::set *modules) +{ + DIR *dir = opendir(path.c_str()); + if (!dir) { + return; + } + struct dirent *entry = NULL; + while ((entry = readdir(dir)) != NULL) { + string n(entry->d_name); + string fn = path + "/" + n; + struct stat st; + int r = ::stat(fn.c_str(), &st); + if (r == 0 && S_ISDIR(st.st_mode)) { + string initfn = fn + "/module.py"; + r = ::stat(initfn.c_str(), &st); + if (r == 0) { + modules->insert(n); + } + } + } + closedir(dir); +} + +void PyModules::list_modules(std::set *modules) +{ + _list_modules(g_conf->mgr_module_path, modules); +} diff --git a/src/mgr/PyModules.h b/src/mgr/PyModules.h index c98f7694bf6..93cbd5be851 100644 --- a/src/mgr/PyModules.h +++ b/src/mgr/PyModules.h @@ -101,6 +101,8 @@ public: void log(const std::string &handle, int level, const std::string &record); + + static void list_modules(std::set *modules); }; #endif diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h index 4547bb4222f..3841840cff7 100644 --- a/src/mon/MgrMap.h +++ b/src/mon/MgrMap.h @@ -25,9 +25,11 @@ class StandbyInfo public: uint64_t gid; std::string name; + std::set available_modules; - StandbyInfo(uint64_t gid_, const std::string &name_) - : gid(gid_), name(name_) + StandbyInfo(uint64_t gid_, const std::string &name_, + std::set& am) + : gid(gid_), name(name_), available_modules(am) {} StandbyInfo() @@ -36,17 +38,21 @@ public: void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); ::encode(gid, bl); ::encode(name, bl); + ::encode(available_modules, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& p) { - DECODE_START(1, p); + DECODE_START(2, p); ::decode(gid, p); ::decode(name, p); + if (struct_v >= 2) { + ::decode(available_modules, p); + } DECODE_FINISH(p); } }; @@ -69,6 +75,7 @@ public: std::map standbys; std::set modules; + std::set available_modules; epoch_t get_epoch() const { return epoch; } entity_addr_t get_active_addr() const { return active_addr; } @@ -76,6 +83,18 @@ public: bool get_available() const { return available; } const std::string &get_active_name() const { return active_name; } + bool all_support_module(const std::string& module) { + if (!available_modules.count(module)) { + return false; + } + for (auto& p : standbys) { + if (!p.second.available_modules.count(module)) { + return false; + } + } + return true; + } + void encode(bufferlist& bl, uint64_t features) const { ENCODE_START(2, 1, bl); @@ -86,6 +105,7 @@ public: ::encode(active_name, bl); ::encode(standbys, bl); ::encode(modules, bl); + ::encode(available_modules, bl); ENCODE_FINISH(bl); } @@ -100,6 +120,7 @@ public: ::decode(standbys, p); if (struct_v >= 2) { ::decode(modules, p); + ::decode(available_modules, p); } DECODE_FINISH(p); } @@ -115,6 +136,11 @@ public: f->open_object_section("standby"); f->dump_int("gid", i.second.gid); f->dump_string("name", i.second.name); + f->open_array_section("available_modules"); + for (auto& j : i.second.available_modules) { + f->dump_string("module", j); + } + f->close_section(); f->close_section(); } f->close_section(); @@ -123,6 +149,11 @@ public: f->dump_string("module", i); } f->close_section(); + f->open_array_section("available_modules"); + for (auto& j : available_modules) { + f->dump_string("module", j); + } + f->close_section(); } static void generate_test_instances(list &l) { diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc index 676cb92e9b1..dfdd0834ed3 100644 --- a/src/mon/MgrMonitor.cc +++ b/src/mon/MgrMonitor.cc @@ -213,6 +213,12 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op) pending_map.available = m->get_available(); updated = true; } + if (pending_map.available_modules != m->get_available_modules()) { + dout(4) << "available_modules " << m->get_available_modules() + << " (was " << pending_map.available_modules << ")" << dendl; + pending_map.available_modules = m->get_available_modules(); + updated = true; + } } else if (pending_map.active_gid == 0) { // There is no currently active daemon, select this one. if (pending_map.standbys.count(m->get_gid())) { @@ -224,14 +230,26 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op) << pending_map.active_name << ")" << dendl; pending_map.active_gid = m->get_gid(); pending_map.active_name = m->get_name(); + pending_map.available_modules = m->get_available_modules(); updated = true; } else { if (pending_map.standbys.count(m->get_gid()) > 0) { dout(10) << "from existing standby " << m->get_gid() << dendl; + if (pending_map.standbys[m->get_gid()].available_modules != + m->get_available_modules()) { + dout(10) << "existing standby " << m->get_gid() << " available_modules " + << m->get_available_modules() << " (was " + << pending_map.standbys[m->get_gid()].available_modules << ")" + << dendl; + pending_map.standbys[m->get_gid()].available_modules = + m->get_available_modules(); + updated = true; + } } else { dout(10) << "new standby " << m->get_gid() << dendl; - pending_map.standbys[m->get_gid()] = {m->get_gid(), m->get_name()}; + pending_map.standbys[m->get_gid()] = {m->get_gid(), m->get_name(), + m->get_available_modules()}; updated = true; } } @@ -603,6 +621,15 @@ bool MgrMonitor::prepare_command(MonOpRequestRef op) r = -EINVAL; goto out; } + string force; + cmd_getval(g_ceph_context, cmdmap, "force", force); + if (!pending_map.all_support_module(module) && + force != "--force") { + ss << "all mgr daemons do not support module '" << module << "', pass " + << "--force to force enablement"; + r = -ENOENT; + goto out; + } pending_map.modules.insert(module); } else if (prefix == "mgr module disable") { string module; diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index 0e5e308ab85..b45d2b7b059 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -952,9 +952,10 @@ COMMAND("mgr fail name=who,type=CephString", \ "treat the named manager daemon as failed", "mgr", "rw", "cli,rest") COMMAND("mgr module ls", "list active mgr modules", "mgr", "r", "cli,rest") -COMMAND("mgr module enable " \ - "name=module,type=CephString", +COMMAND("mgr module enable " \ + "name=module,type=CephString " \ + "name=force,type=CephChoices,strings=--force,req=false", "enable mgr module", "mgr", "rw", "cli,rest") -COMMAND("mgr module disable " \ +COMMAND("mgr module disable " \ "name=module,type=CephString", - "enable mgr module", "mgr", "rw", "cli,rest") + "disable mgr module", "mgr", "rw", "cli,rest") -- 2.39.5