From: Sage Weil Date: Tue, 27 Jun 2017 19:58:44 +0000 (-0400) Subject: mon,mgr: manage mgr module list in mgrmap X-Git-Tag: v12.1.1~195^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=16fcee1f71fce3a2365d17dfef0b1459f8da1948;p=ceph.git mon,mgr: manage mgr module list in mgrmap Kill old mgr_modules option. Add new mgr_initial_modules option, on the mon, for the initial cluster mgrmap. Add ls, enable, disable commands. Respawn mgr if the module list changes. In the future we could enable new modules without a full restart, but disabling probably requires (and is best handled by) a respawn. Signed-off-by: Sage Weil --- diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index 6521d9df995..0f6b428c3cd 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -712,6 +712,9 @@ function test_mon_misc() ceph_watch_wait "$mymsg" ceph mgr dump + ceph mgr module ls + ceph mgr module enable foo + ceph mgr module disable foo ceph mon metadata a ceph mon metadata diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 1c94cd0f1b1..d7096010c63 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1730,7 +1730,7 @@ OPTION(rgw_shard_warning_threshold, OPT_DOUBLE, 90) // pct of safe max OPTION(rgw_swift_versioning_enabled, OPT_BOOL, false) // whether swift object versioning feature is enabled OPTION(mgr_module_path, OPT_STR, CEPH_PKGLIBDIR "/mgr") // where to load python modules from -OPTION(mgr_modules, OPT_STR, "restful status") // Which modules to load +OPTION(mgr_initial_modules, OPT_STR, "restful status") // Which modules to load OPTION(mgr_data, OPT_STR, "/var/lib/ceph/mgr/$cluster-$id") // where to find keyring etc OPTION(mgr_tick_period, OPT_INT, 2) // How frequently to tick OPTION(mgr_stats_period, OPT_INT, 5) // How frequently clients send stats diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc index 0030fc706c5..ffc8e008ab0 100644 --- a/src/mgr/Mgr.cc +++ b/src/mgr/Mgr.cc @@ -568,11 +568,22 @@ void Mgr::handle_fs_map(MFSMap* m) daemon_state.cull(CEPH_ENTITY_TYPE_MDS, names_exist); } -void Mgr::got_mgr_map(const MgrMap& m) +bool Mgr::got_mgr_map(const MgrMap& m) { Mutex::Locker l(lock); dout(10) << m << dendl; + + set old_modules; + cluster_state.with_mgrmap([&](const MgrMap& m) { + old_modules = m.modules; + }); + if (m.modules != old_modules) { + derr << "mgrmap module list changed to (" << m.modules << "), respawn" + << dendl; + return true; + } cluster_state.set_mgr_map(m); + return false; } void Mgr::handle_mgr_digest(MMgrDigest* m) diff --git a/src/mgr/Mgr.h b/src/mgr/Mgr.h index d6e7bc629a7..3638f5b1688 100644 --- a/src/mgr/Mgr.h +++ b/src/mgr/Mgr.h @@ -85,7 +85,7 @@ public: void handle_osd_map(); void handle_log(MLog *m); - void got_mgr_map(const MgrMap& m); + bool got_mgr_map(const MgrMap& m); bool ms_dispatch(Message *m); diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc index 5d2e735bb85..bc7422ba630 100644 --- a/src/mgr/MgrStandby.cc +++ b/src/mgr/MgrStandby.cc @@ -288,7 +288,10 @@ void MgrStandby::handle_mgr_map(MMgrMap* mmap) dout(1) << "I am now active" << dendl; } else { dout(10) << "I was already active" << dendl; - active_mgr->got_mgr_map(map); + bool need_respawn = active_mgr->got_mgr_map(map); + if (need_respawn) { + respawn(); + } } } else { if (active_mgr != nullptr) { diff --git a/src/mgr/PyModules.cc b/src/mgr/PyModules.cc index 6d5788a3033..7d091a253e1 100644 --- a/src/mgr/PyModules.cc +++ b/src/mgr/PyModules.cc @@ -15,7 +15,6 @@ #include "PyState.h" #include "Gil.h" -#include #include "common/errno.h" #include "include/stringify.h" @@ -363,8 +362,11 @@ int PyModules::init() std::list failed_modules; // Load python code - boost::tokenizer<> tok(g_conf->mgr_modules); - for(const auto& module_name : tok) { + set ls; + cluster_state.with_mgrmap([&](const MgrMap& m) { + ls = m.modules; + }); + for (const auto& module_name : ls) { dout(1) << "Loading python module '" << module_name << "'" << dendl; auto mod = std::unique_ptr(new MgrPyModule(module_name, sys_path, pMainThreadState)); int r = mod->load(); diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h index 5a7d33397e6..4547bb4222f 100644 --- a/src/mon/MgrMap.h +++ b/src/mon/MgrMap.h @@ -68,6 +68,8 @@ public: std::map standbys; + std::set modules; + epoch_t get_epoch() const { return epoch; } entity_addr_t get_active_addr() const { return active_addr; } uint64_t get_active_gid() const { return active_gid; } @@ -76,25 +78,29 @@ public: void encode(bufferlist& bl, uint64_t features) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); ::encode(epoch, bl); ::encode(active_addr, bl, features); ::encode(active_gid, bl); ::encode(available, bl); ::encode(active_name, bl); ::encode(standbys, bl); + ::encode(modules, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& p) { - DECODE_START(1, p); + DECODE_START(2, p); ::decode(epoch, p); ::decode(active_addr, p); ::decode(active_gid, p); ::decode(available, p); ::decode(active_name, p); ::decode(standbys, p); + if (struct_v >= 2) { + ::decode(modules, p); + } DECODE_FINISH(p); } @@ -112,6 +118,11 @@ public: f->close_section(); } f->close_section(); + f->open_array_section("modules"); + for (auto& i : modules) { + f->dump_string("module", i); + } + f->close_section(); } static void generate_test_instances(list &l) { diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc index 863672afe7f..676cb92e9b1 100644 --- a/src/mon/MgrMonitor.cc +++ b/src/mon/MgrMonitor.cc @@ -11,6 +11,8 @@ * Foundation. See file COPYING. */ +#include + #include "messages/MMgrBeacon.h" #include "messages/MMgrMap.h" #include "messages/MMgrDigest.h" @@ -35,6 +37,11 @@ static ostream& _prefix(std::ostream *_dout, Monitor *mon, void MgrMonitor::create_initial() { + boost::tokenizer<> tok(g_conf->mgr_initial_modules); + for (auto& m : tok) { + pending_map.modules.insert(m); + } + dout(10) << __func__ << " initial modules " << pending_map.modules << dendl; } void MgrMonitor::update_from_paxos(bool *need_bootstrap) @@ -500,6 +507,13 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) f->dump_object("mgrmap", m); } f->flush(rdata); + } else if (prefix == "mgr module ls") { + f->open_array_section("modules"); + for (auto& p : map.modules) { + f->dump_string("module", p); + } + f->close_section(); + f->flush(rdata); } else { return false; } @@ -531,6 +545,10 @@ bool MgrMonitor::prepare_command(MonOpRequestRef op) return true; } + string format; + cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain")); + boost::scoped_ptr f(Formatter::create(format)); + string prefix; cmd_getval(g_ceph_context, cmdmap, "prefix", prefix); @@ -578,10 +596,27 @@ bool MgrMonitor::prepare_command(MonOpRequestRef op) if (changed && pending_map.active_gid == 0) { promote_standby(); } + } else if (prefix == "mgr module enable") { + string module; + cmd_getval(g_ceph_context, cmdmap, "module", module); + if (module.empty()) { + r = -EINVAL; + goto out; + } + pending_map.modules.insert(module); + } else if (prefix == "mgr module disable") { + string module; + cmd_getval(g_ceph_context, cmdmap, "module", module); + if (module.empty()) { + r = -EINVAL; + goto out; + } + pending_map.modules.erase(module); } else { r = -ENOSYS; } +out: dout(4) << __func__ << " done, r=" << r << dendl; /* Compose response */ string rs; diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index e5e3cd98bef..0e5e308ab85 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -950,3 +950,11 @@ COMMAND("mgr dump " \ "mgr", "r", "cli,rest") COMMAND("mgr fail name=who,type=CephString", \ "treat the named manager daemon as failed", "mgr", "rw", "cli,rest") +COMMAND("mgr module ls", + "list active mgr modules", "mgr", "r", "cli,rest") +COMMAND("mgr module enable " \ + "name=module,type=CephString", + "enable mgr module", "mgr", "rw", "cli,rest") +COMMAND("mgr module disable " \ + "name=module,type=CephString", + "enable mgr module", "mgr", "rw", "cli,rest") diff --git a/src/test/mgr/mgr-dashboard-smoke.sh b/src/test/mgr/mgr-dashboard-smoke.sh index 3c953b99620..5c52215beba 100755 --- a/src/test/mgr/mgr-dashboard-smoke.sh +++ b/src/test/mgr/mgr-dashboard-smoke.sh @@ -25,7 +25,9 @@ function run() { FSID=$(uuidgen) export CEPH_ARGS CEPH_ARGS+="--fsid=$FSID --auth-supported=none " - CEPH_ARGS+="--mon-initial-members=a --mon-host=$MON" + CEPH_ARGS+="--mon-initial-members=a --mon-host=$MON " + CEPH_ARGS+="--mgr-initial-modules=dashbaord " + CEPH_ARGS+="--mon-host=$MON" run_mon $dir a --public-addr $MON || return 1 ) @@ -33,7 +35,6 @@ function run() { export CEPH_ARGS="--mon_host $MON " ceph config-key put mgr/x/dashboard/server_port 7001 MGR_ARGS+="--mgr_module_path=${CEPH_ROOT}/src/pybind/mgr " - MGR_ARGS+="--mgr_modules=dashboard " run_mgr $dir x ${MGR_ARGS} || return 1 tries=0 diff --git a/src/vstart.sh b/src/vstart.sh index 6eb3f263c05..12f6c540c79 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -458,7 +458,6 @@ $CMDSDEBUG mds root ino gid = `id -g` $extra_conf [mgr] - mgr modules = restful status dashboard mgr data = $CEPH_DEV_DIR/mgr.\$id mgr module path = $MGR_PYTHON_PATH mon reweight min pgs per osd = 4 @@ -497,6 +496,7 @@ $COSDMEMSTORE $COSDSHORT $extra_conf [mon] + mgr initial modules = restful status dashboard mon pg warn min per osd = 3 mon osd allow primary affinity = true mon reweight min pgs per osd = 4