mon,mgr: manage mgr module list in mgrmap

Kill old mgr_modules option.

Add new mgr_initial_modules option, on the mon, for the initial cluster
mgrmap.

Add ls, enable, disable commands.

Respawn mgr if the module list changes.  In the future we could enable
new modules without a full restart, but disabling probably requires (and
is best handled by) a respawn.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh
index 6521d9df9951b93d5f55a0995a85344d42ffdf47..0f6b428c3cde7ac952c1690f196a196032db6e1c 100755 (executable)
--- a/qa/workunits/cephtool/test.sh
+++ b/qa/workunits/cephtool/test.sh
@@ -712,6 +712,9 @@ function test_mon_misc()
   ceph_watch_wait "$mymsg"
 
   ceph mgr dump
+  ceph mgr module ls
+  ceph mgr module enable foo
+  ceph mgr module disable foo
 
   ceph mon metadata a
   ceph mon metadata

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 1c94cd0f1b18120a470193c1d166afae500eec6f..d7096010c63e4614aaab69ace14d0292ac27b669 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1730,7 +1730,7 @@ OPTION(rgw_shard_warning_threshold, OPT_DOUBLE, 90) // pct of safe max
 OPTION(rgw_swift_versioning_enabled, OPT_BOOL, false) // whether swift object versioning feature is enabled
 
 OPTION(mgr_module_path, OPT_STR, CEPH_PKGLIBDIR "/mgr") // where to load python modules from
-OPTION(mgr_modules, OPT_STR, "restful status")  // Which modules to load
+OPTION(mgr_initial_modules, OPT_STR, "restful status")  // Which modules to load
 OPTION(mgr_data, OPT_STR, "/var/lib/ceph/mgr/$cluster-$id") // where to find keyring etc
 OPTION(mgr_tick_period, OPT_INT, 2)  // How frequently to tick
 OPTION(mgr_stats_period, OPT_INT, 5) // How frequently clients send stats

diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc
index 0030fc706c59b65467e25bb683479f490905a5e8..ffc8e008ab025de23357ae0c1e49fc0b1612c27c 100644 (file)
--- a/src/mgr/Mgr.cc
+++ b/src/mgr/Mgr.cc
@@ -568,11 +568,22 @@ void Mgr::handle_fs_map(MFSMap* m)
   daemon_state.cull(CEPH_ENTITY_TYPE_MDS, names_exist);
 }
 
-void Mgr::got_mgr_map(const MgrMap& m)
+bool Mgr::got_mgr_map(const MgrMap& m)
 {
   Mutex::Locker l(lock);
   dout(10) << m << dendl;
+
+  set<string> old_modules;
+  cluster_state.with_mgrmap([&](const MgrMap& m) {
+      old_modules = m.modules;
+    });
+  if (m.modules != old_modules) {
+    derr << "mgrmap module list changed to (" << m.modules << "), respawn"
+        << dendl;
+    return true;
+  }
   cluster_state.set_mgr_map(m);
+  return false;
 }
 
 void Mgr::handle_mgr_digest(MMgrDigest* m)

diff --git a/src/mgr/Mgr.h b/src/mgr/Mgr.h
index d6e7bc629a70b454bb5bcf36261844a7e295cd05..3638f5b16884f55e7860076b1de0a1713bc195ee 100644 (file)
--- a/src/mgr/Mgr.h
+++ b/src/mgr/Mgr.h
@@ -85,7 +85,7 @@ public:
   void handle_osd_map();
   void handle_log(MLog *m);
 
-  void got_mgr_map(const MgrMap& m);
+  bool got_mgr_map(const MgrMap& m);
 
   bool ms_dispatch(Message *m);
 

diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc
index 5d2e735bb85064b772b29fc5d9fe6783312ab38f..bc7422ba63051f093b6f0b702e6db5d5071f3740 100644 (file)
--- a/src/mgr/MgrStandby.cc
+++ b/src/mgr/MgrStandby.cc
@@ -288,7 +288,10 @@ void MgrStandby::handle_mgr_map(MMgrMap* mmap)
       dout(1) << "I am now active" << dendl;
     } else {
       dout(10) << "I was already active" << dendl;
-      active_mgr->got_mgr_map(map);
+      bool need_respawn = active_mgr->got_mgr_map(map);
+      if (need_respawn) {
+       respawn();
+      }
     }
   } else {
     if (active_mgr != nullptr) {

diff --git a/src/mgr/PyModules.cc b/src/mgr/PyModules.cc
index 6d5788a303305bd23a30b0251e9be385801967ae..7d091a253e177ae0d0bc4cd5438072821b5db177 100644 (file)
--- a/src/mgr/PyModules.cc
+++ b/src/mgr/PyModules.cc
@@ -15,7 +15,6 @@
 #include "PyState.h"
 #include "Gil.h"
 
-#include <boost/tokenizer.hpp>
 #include "common/errno.h"
 #include "include/stringify.h"
 
@@ -363,8 +362,11 @@ int PyModules::init()
   std::list<std::string> failed_modules;
 
   // Load python code
-  boost::tokenizer<> tok(g_conf->mgr_modules);
-  for(const auto& module_name : tok) {
+  set<string> ls;
+  cluster_state.with_mgrmap([&](const MgrMap& m) {
+      ls = m.modules;
+    });
+  for (const auto& module_name : ls) {
     dout(1) << "Loading python module '" << module_name << "'" << dendl;
     auto mod = std::unique_ptr<MgrPyModule>(new MgrPyModule(module_name, sys_path, pMainThreadState));
     int r = mod->load();

diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h
index 5a7d33397e6410d2c1e65b4e05ae18470d8eb426..4547bb4222f520504f260d3f49233970270128ba 100644 (file)
--- a/src/mon/MgrMap.h
+++ b/src/mon/MgrMap.h
@@ -68,6 +68,8 @@ public:
 
   std::map<uint64_t, StandbyInfo> standbys;
 
+  std::set<std::string> modules;
+
   epoch_t get_epoch() const { return epoch; }
   entity_addr_t get_active_addr() const { return active_addr; }
   uint64_t get_active_gid() const { return active_gid; }
@@ -76,25 +78,29 @@ public:
 
   void encode(bufferlist& bl, uint64_t features) const
   {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(epoch, bl);
     ::encode(active_addr, bl, features);
     ::encode(active_gid, bl);
     ::encode(available, bl);
     ::encode(active_name, bl);
     ::encode(standbys, bl);
+    ::encode(modules, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& p)
   {
-    DECODE_START(1, p);
+    DECODE_START(2, p);
     ::decode(epoch, p);
     ::decode(active_addr, p);
     ::decode(active_gid, p);
     ::decode(available, p);
     ::decode(active_name, p);
     ::decode(standbys, p);
+    if (struct_v >= 2) {
+      ::decode(modules, p);
+    }
     DECODE_FINISH(p);
   }
 
@@ -112,6 +118,11 @@ public:
       f->close_section();
     }
     f->close_section();
+    f->open_array_section("modules");
+    for (auto& i : modules) {
+      f->dump_string("module", i);
+    }
+    f->close_section();
   }
 
   static void generate_test_instances(list<MgrMap*> &l) {

diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc
index 863672afe7f5d7ccb29ec14a5a53bd6aa2bf32a5..676cb92e9b10cb116c6217c0d095931797cf393c 100644 (file)
--- a/src/mon/MgrMonitor.cc
+++ b/src/mon/MgrMonitor.cc
@@ -11,6 +11,8 @@
  * Foundation.  See file COPYING.
  */
 
+#include <boost/tokenizer.hpp>
+
 #include "messages/MMgrBeacon.h"
 #include "messages/MMgrMap.h"
 #include "messages/MMgrDigest.h"
@@ -35,6 +37,11 @@ static ostream& _prefix(std::ostream *_dout, Monitor *mon,
 
 void MgrMonitor::create_initial()
 {
+  boost::tokenizer<> tok(g_conf->mgr_initial_modules);
+  for (auto& m : tok) {
+    pending_map.modules.insert(m);
+  }
+  dout(10) << __func__ << " initial modules " << pending_map.modules << dendl;
 }
 
 void MgrMonitor::update_from_paxos(bool *need_bootstrap)
@@ -500,6 +507,13 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op)
       f->dump_object("mgrmap", m);
     }
     f->flush(rdata);
+  } else if (prefix == "mgr module ls") {
+    f->open_array_section("modules");
+    for (auto& p : map.modules) {
+      f->dump_string("module", p);
+    }
+    f->close_section();
+    f->flush(rdata);
   } else {
     return false;
   }
@@ -531,6 +545,10 @@ bool MgrMonitor::prepare_command(MonOpRequestRef op)
     return true;
   }
 
+  string format;
+  cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
   string prefix;
   cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
 
@@ -578,10 +596,27 @@ bool MgrMonitor::prepare_command(MonOpRequestRef op)
     if (changed && pending_map.active_gid == 0) {
       promote_standby();
     }
+  } else if (prefix == "mgr module enable") {
+    string module;
+    cmd_getval(g_ceph_context, cmdmap, "module", module);
+    if (module.empty()) {
+      r = -EINVAL;
+      goto out;
+    }
+    pending_map.modules.insert(module);
+  } else if (prefix == "mgr module disable") {
+    string module;
+    cmd_getval(g_ceph_context, cmdmap, "module", module);
+    if (module.empty()) {
+      r = -EINVAL;
+      goto out;
+    }
+    pending_map.modules.erase(module);
   } else {
     r = -ENOSYS;
   }
 
+out:
   dout(4) << __func__ << " done, r=" << r << dendl;
   /* Compose response */
   string rs;

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index e5e3cd98befb01b8551e7790a73fe266dda0e76f..0e5e308ab851e2a68e57afcefddfd7adcc37cbac 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -950,3 +950,11 @@ COMMAND("mgr dump "                                     \
        "mgr", "r", "cli,rest")
 COMMAND("mgr fail name=who,type=CephString", \
        "treat the named manager daemon as failed", "mgr", "rw", "cli,rest")
+COMMAND("mgr module ls",
+       "list active mgr modules", "mgr", "r", "cli,rest")
+COMMAND("mgr module enable "                   \
+       "name=module,type=CephString",
+       "enable mgr module", "mgr", "rw", "cli,rest")
+COMMAND("mgr module disable "                  \
+       "name=module,type=CephString",
+       "enable mgr module", "mgr", "rw", "cli,rest")

diff --git a/src/test/mgr/mgr-dashboard-smoke.sh b/src/test/mgr/mgr-dashboard-smoke.sh
index 3c953b996202532dbb1cd74d755a772838e5e177..5c52215beba4d1f8cc1e14431d89854f8ffca2d1 100755 (executable)
--- a/src/test/mgr/mgr-dashboard-smoke.sh
+++ b/src/test/mgr/mgr-dashboard-smoke.sh
@@ -25,7 +25,9 @@ function run() {
         FSID=$(uuidgen) 
         export CEPH_ARGS
         CEPH_ARGS+="--fsid=$FSID --auth-supported=none "
-        CEPH_ARGS+="--mon-initial-members=a --mon-host=$MON"
+        CEPH_ARGS+="--mon-initial-members=a --mon-host=$MON "
+        CEPH_ARGS+="--mgr-initial-modules=dashbaord "
+       CEPH_ARGS+="--mon-host=$MON"
         run_mon $dir a --public-addr $MON || return 1
     )
 
@@ -33,7 +35,6 @@ function run() {
     export CEPH_ARGS="--mon_host $MON "
     ceph config-key put mgr/x/dashboard/server_port 7001
     MGR_ARGS+="--mgr_module_path=${CEPH_ROOT}/src/pybind/mgr "
-    MGR_ARGS+="--mgr_modules=dashboard "
     run_mgr $dir x ${MGR_ARGS} || return 1
 
     tries=0

diff --git a/src/vstart.sh b/src/vstart.sh
index 6eb3f263c050b95ce1074873b1dd052f55690daa..12f6c540c79b205c1adcc3c47e1b3057112ec3bb 100755 (executable)
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -458,7 +458,6 @@ $CMDSDEBUG
         mds root ino gid = `id -g`
 $extra_conf
 [mgr]
-        mgr modules = restful status dashboard
         mgr data = $CEPH_DEV_DIR/mgr.\$id
         mgr module path = $MGR_PYTHON_PATH
         mon reweight min pgs per osd = 4
@@ -497,6 +496,7 @@ $COSDMEMSTORE
 $COSDSHORT
 $extra_conf
 [mon]
+        mgr initial modules = restful status dashboard
         mon pg warn min per osd = 3
         mon osd allow primary affinity = true
         mon reweight min pgs per osd = 4