From: Sage Weil Date: Tue, 27 Jun 2017 19:58:44 +0000 (-0400) Subject: mon,mgr: manage mgr module list in mgrmap X-Git-Tag: v12.1.1~195^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=16fcee1f71fce3a2365d17dfef0b1459f8da1948;p=ceph.git mon,mgr: manage mgr module list in mgrmap Kill old mgr_modules option. Add new mgr_initial_modules option, on the mon, for the initial cluster mgrmap. Add ls, enable, disable commands. Respawn mgr if the module list changes. In the future we could enable new modules without a full restart, but disabling probably requires (and is best handled by) a respawn. Signed-off-by: Sage Weil --- diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index 6521d9df9951..0f6b428c3cde 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -712,6 +712,9 @@ function test_mon_misc() ceph_watch_wait "$mymsg" ceph mgr dump + ceph mgr module ls + ceph mgr module enable foo + ceph mgr module disable foo ceph mon metadata a ceph mon metadata diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 1c94cd0f1b18..d7096010c63e 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1730,7 +1730,7 @@ OPTION(rgw_shard_warning_threshold, OPT_DOUBLE, 90) // pct of safe max OPTION(rgw_swift_versioning_enabled, OPT_BOOL, false) // whether swift object versioning feature is enabled OPTION(mgr_module_path, OPT_STR, CEPH_PKGLIBDIR "/mgr") // where to load python modules from -OPTION(mgr_modules, OPT_STR, "restful status") // Which modules to load +OPTION(mgr_initial_modules, OPT_STR, "restful status") // Which modules to load OPTION(mgr_data, OPT_STR, "/var/lib/ceph/mgr/$cluster-$id") // where to find keyring etc OPTION(mgr_tick_period, OPT_INT, 2) // How frequently to tick OPTION(mgr_stats_period, OPT_INT, 5) // How frequently clients send stats diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc index 0030fc706c59..ffc8e008ab02 100644 --- a/src/mgr/Mgr.cc +++ b/src/mgr/Mgr.cc @@ -568,11 +568,22 @@ void Mgr::handle_fs_map(MFSMap* m) daemon_state.cull(CEPH_ENTITY_TYPE_MDS, names_exist); } -void Mgr::got_mgr_map(const MgrMap& m) +bool Mgr::got_mgr_map(const MgrMap& m) { Mutex::Locker l(lock); dout(10) << m << dendl; + + set old_modules; + cluster_state.with_mgrmap([&](const MgrMap& m) { + old_modules = m.modules; + }); + if (m.modules != old_modules) { + derr << "mgrmap module list changed to (" << m.modules << "), respawn" + << dendl; + return true; + } cluster_state.set_mgr_map(m); + return false; } void Mgr::handle_mgr_digest(MMgrDigest* m) diff --git a/src/mgr/Mgr.h b/src/mgr/Mgr.h index d6e7bc629a70..3638f5b16884 100644 --- a/src/mgr/Mgr.h +++ b/src/mgr/Mgr.h @@ -85,7 +85,7 @@ public: void handle_osd_map(); void handle_log(MLog *m); - void got_mgr_map(const MgrMap& m); + bool got_mgr_map(const MgrMap& m); bool ms_dispatch(Message *m); diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc index 5d2e735bb850..bc7422ba6305 100644 --- a/src/mgr/MgrStandby.cc +++ b/src/mgr/MgrStandby.cc @@ -288,7 +288,10 @@ void MgrStandby::handle_mgr_map(MMgrMap* mmap) dout(1) << "I am now active" << dendl; } else { dout(10) << "I was already active" << dendl; - active_mgr->got_mgr_map(map); + bool need_respawn = active_mgr->got_mgr_map(map); + if (need_respawn) { + respawn(); + } } } else { if (active_mgr != nullptr) { diff --git a/src/mgr/PyModules.cc b/src/mgr/PyModules.cc index 6d5788a30330..7d091a253e17 100644 --- a/src/mgr/PyModules.cc +++ b/src/mgr/PyModules.cc @@ -15,7 +15,6 @@ #include "PyState.h" #include "Gil.h" -#include #include "common/errno.h" #include "include/stringify.h" @@ -363,8 +362,11 @@ int PyModules::init() std::list failed_modules; // Load python code - boost::tokenizer<> tok(g_conf->mgr_modules); - for(const auto& module_name : tok) { + set ls; + cluster_state.with_mgrmap([&](const MgrMap& m) { + ls = m.modules; + }); + for (const auto& module_name : ls) { dout(1) << "Loading python module '" << module_name << "'" << dendl; auto mod = std::unique_ptr(new MgrPyModule(module_name, sys_path, pMainThreadState)); int r = mod->load(); diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h index 5a7d33397e64..4547bb4222f5 100644 --- a/src/mon/MgrMap.h +++ b/src/mon/MgrMap.h @@ -68,6 +68,8 @@ public: std::map standbys; + std::set modules; + epoch_t get_epoch() const { return epoch; } entity_addr_t get_active_addr() const { return active_addr; } uint64_t get_active_gid() const { return active_gid; } @@ -76,25 +78,29 @@ public: void encode(bufferlist& bl, uint64_t features) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); ::encode(epoch, bl); ::encode(active_addr, bl, features); ::encode(active_gid, bl); ::encode(available, bl); ::encode(active_name, bl); ::encode(standbys, bl); + ::encode(modules, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& p) { - DECODE_START(1, p); + DECODE_START(2, p); ::decode(epoch, p); ::decode(active_addr, p); ::decode(active_gid, p); ::decode(available, p); ::decode(active_name, p); ::decode(standbys, p); + if (struct_v >= 2) { + ::decode(modules, p); + } DECODE_FINISH(p); } @@ -112,6 +118,11 @@ public: f->close_section(); } f->close_section(); + f->open_array_section("modules"); + for (auto& i : modules) { + f->dump_string("module", i); + } + f->close_section(); } static void generate_test_instances(list &l) { diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc index 863672afe7f5..676cb92e9b10 100644 --- a/src/mon/MgrMonitor.cc +++ b/src/mon/MgrMonitor.cc @@ -11,6 +11,8 @@ * Foundation. See file COPYING. */ +#include + #include "messages/MMgrBeacon.h" #include "messages/MMgrMap.h" #include "messages/MMgrDigest.h" @@ -35,6 +37,11 @@ static ostream& _prefix(std::ostream *_dout, Monitor *mon, void MgrMonitor::create_initial() { + boost::tokenizer<> tok(g_conf->mgr_initial_modules); + for (auto& m : tok) { + pending_map.modules.insert(m); + } + dout(10) << __func__ << " initial modules " << pending_map.modules << dendl; } void MgrMonitor::update_from_paxos(bool *need_bootstrap) @@ -500,6 +507,13 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) f->dump_object("mgrmap", m); } f->flush(rdata); + } else if (prefix == "mgr module ls") { + f->open_array_section("modules"); + for (auto& p : map.modules) { + f->dump_string("module", p); + } + f->close_section(); + f->flush(rdata); } else { return false; } @@ -531,6 +545,10 @@ bool MgrMonitor::prepare_command(MonOpRequestRef op) return true; } + string format; + cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain")); + boost::scoped_ptr f(Formatter::create(format)); + string prefix; cmd_getval(g_ceph_context, cmdmap, "prefix", prefix); @@ -578,10 +596,27 @@ bool MgrMonitor::prepare_command(MonOpRequestRef op) if (changed && pending_map.active_gid == 0) { promote_standby(); } + } else if (prefix == "mgr module enable") { + string module; + cmd_getval(g_ceph_context, cmdmap, "module", module); + if (module.empty()) { + r = -EINVAL; + goto out; + } + pending_map.modules.insert(module); + } else if (prefix == "mgr module disable") { + string module; + cmd_getval(g_ceph_context, cmdmap, "module", module); + if (module.empty()) { + r = -EINVAL; + goto out; + } + pending_map.modules.erase(module); } else { r = -ENOSYS; } +out: dout(4) << __func__ << " done, r=" << r << dendl; /* Compose response */ string rs; diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index e5e3cd98befb..0e5e308ab851 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -950,3 +950,11 @@ COMMAND("mgr dump " \ "mgr", "r", "cli,rest") COMMAND("mgr fail name=who,type=CephString", \ "treat the named manager daemon as failed", "mgr", "rw", "cli,rest") +COMMAND("mgr module ls", + "list active mgr modules", "mgr", "r", "cli,rest") +COMMAND("mgr module enable " \ + "name=module,type=CephString", + "enable mgr module", "mgr", "rw", "cli,rest") +COMMAND("mgr module disable " \ + "name=module,type=CephString", + "enable mgr module", "mgr", "rw", "cli,rest") diff --git a/src/test/mgr/mgr-dashboard-smoke.sh b/src/test/mgr/mgr-dashboard-smoke.sh index 3c953b996202..5c52215beba4 100755 --- a/src/test/mgr/mgr-dashboard-smoke.sh +++ b/src/test/mgr/mgr-dashboard-smoke.sh @@ -25,7 +25,9 @@ function run() { FSID=$(uuidgen) export CEPH_ARGS CEPH_ARGS+="--fsid=$FSID --auth-supported=none " - CEPH_ARGS+="--mon-initial-members=a --mon-host=$MON" + CEPH_ARGS+="--mon-initial-members=a --mon-host=$MON " + CEPH_ARGS+="--mgr-initial-modules=dashbaord " + CEPH_ARGS+="--mon-host=$MON" run_mon $dir a --public-addr $MON || return 1 ) @@ -33,7 +35,6 @@ function run() { export CEPH_ARGS="--mon_host $MON " ceph config-key put mgr/x/dashboard/server_port 7001 MGR_ARGS+="--mgr_module_path=${CEPH_ROOT}/src/pybind/mgr " - MGR_ARGS+="--mgr_modules=dashboard " run_mgr $dir x ${MGR_ARGS} || return 1 tries=0 diff --git a/src/vstart.sh b/src/vstart.sh index 6eb3f263c050..12f6c540c79b 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -458,7 +458,6 @@ $CMDSDEBUG mds root ino gid = `id -g` $extra_conf [mgr] - mgr modules = restful status dashboard mgr data = $CEPH_DEV_DIR/mgr.\$id mgr module path = $MGR_PYTHON_PATH mon reweight min pgs per osd = 4 @@ -497,6 +496,7 @@ $COSDMEMSTORE $COSDSHORT $extra_conf [mon] + mgr initial modules = restful status dashboard mon pg warn min per osd = 3 mon osd allow primary affinity = true mon reweight min pgs per osd = 4