mon,mgr: track available modules in MgrMap too

Require --force to enable a module that isn't reported as available.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh
index 0f6b428c3cde7ac952c1690f196a196032db6e1c..bbe314b7b14ce4169172a629febd35e6293666ea 100755 (executable)
--- a/qa/workunits/cephtool/test.sh
+++ b/qa/workunits/cephtool/test.sh
@@ -713,8 +713,11 @@ function test_mon_misc()
 
   ceph mgr dump
   ceph mgr module ls
-  ceph mgr module enable foo
-  ceph mgr module disable foo
+  ceph mgr module enable restful
+  expect_false ceph mgr module enable foodne
+  ceph mgr module enable foodne --force
+  ceph mgr module disable foodne
+  ceph mgr module disable foodnebizbangbash
 
   ceph mon metadata a
   ceph mon metadata

diff --git a/src/messages/MMgrBeacon.h b/src/messages/MMgrBeacon.h
index 2962d9c9a1f9149f85b04bf7e89a349cd22cfa51..ab55a9642a216a1dc8585524f4acc1a42f8179e1 100644 (file)
--- a/src/messages/MMgrBeacon.h
+++ b/src/messages/MMgrBeacon.h
@@ -22,7 +22,7 @@
 
 class MMgrBeacon : public PaxosServiceMessage {
 
-  static const int HEAD_VERSION = 2;
+  static const int HEAD_VERSION = 3;
   static const int COMPAT_VERSION = 1;
 
 protected:
@@ -31,6 +31,7 @@ protected:
   bool available;
   std::string name;
   uuid_d fsid;
+  std::set<std::string> available_modules;
 
 public:
   MMgrBeacon()
@@ -40,10 +41,11 @@ public:
   }
 
   MMgrBeacon(const uuid_d& fsid_, uint64_t gid_, const std::string &name_,
-             entity_addr_t server_addr_, bool available_)
+             entity_addr_t server_addr_, bool available_,
+            const std::set<std::string>& module_list)
     : PaxosServiceMessage(MSG_MGR_BEACON, 0, HEAD_VERSION, COMPAT_VERSION),
       gid(gid_), server_addr(server_addr_), available(available_), name(name_),
-      fsid(fsid_)
+      fsid(fsid_), available_modules(module_list)
   {
   }
 
@@ -52,6 +54,7 @@ public:
   bool get_available() const { return available; }
   const std::string& get_name() const { return name; }
   const uuid_d& get_fsid() const { return fsid; }
+  std::set<std::string>& get_available_modules() { return available_modules; }
 
 private:
   ~MMgrBeacon() override {}
@@ -62,7 +65,8 @@ public:
 
   void print(ostream& out) const override {
     out << get_type_name() << " mgr." << name << "(" << fsid << ","
-       << gid << ", " << server_addr << ", " << available << ")";
+       << gid << ", " << server_addr << ", " << available
+       << ")";
   }
 
   void encode_payload(uint64_t features) override {
@@ -72,6 +76,7 @@ public:
     ::encode(available, payload);
     ::encode(name, payload);
     ::encode(fsid, payload);
+    ::encode(available_modules, payload);
   }
   void decode_payload() override {
     bufferlist::iterator p = payload.begin();
@@ -83,6 +88,9 @@ public:
     if (header.version >= 2) {
       ::decode(fsid, p);
     }
+    if (header.version >= 3) {
+      ::decode(available_modules, p);
+    }
   }
 };
 

diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc
index bc7422ba63051f093b6f0b702e6db5d5071f3740..a3c3234a20821b1659109d05885018ff0f5496c6 100644 (file)
--- a/src/mgr/MgrStandby.cc
+++ b/src/mgr/MgrStandby.cc
@@ -148,16 +148,20 @@ void MgrStandby::send_beacon()
 {
   assert(lock.is_locked_by_me());
   dout(1) << state_str() << dendl;
-  dout(10) << "sending beacon as gid " << monc.get_global_id() << dendl;
 
+  set<string> modules;
+  PyModules::list_modules(&modules);
   bool available = active_mgr != nullptr && active_mgr->is_initialized();
   auto addr = available ? active_mgr->get_server_addr() : entity_addr_t();
+  dout(10) << "sending beacon as gid " << monc.get_global_id()
+          << " modules " << modules << dendl;
+
   MMgrBeacon *m = new MMgrBeacon(monc.get_fsid(),
                                 monc.get_global_id(),
                                  g_conf->name.get_id(),
                                  addr,
-                                 available);
-                                 
+                                 available,
+                                modules);
   monc.send_mon_message(m);
 }
 

diff --git a/src/mgr/PyModules.cc b/src/mgr/PyModules.cc
index 7d091a253e177ae0d0bc4cd5438072821b5db177..96b2698e3c39f8a7208eb1586c171685b49e9aa7 100644 (file)
--- a/src/mgr/PyModules.cc
+++ b/src/mgr/PyModules.cc
@@ -674,3 +674,32 @@ PyObject *PyModules::get_context()
   return capsule;
 }
 
+static void _list_modules(
+  const std::string path,
+  std::set<std::string> *modules)
+{
+  DIR *dir = opendir(path.c_str());
+  if (!dir) {
+    return;
+  }
+  struct dirent *entry = NULL;
+  while ((entry = readdir(dir)) != NULL) {
+    string n(entry->d_name);
+    string fn = path + "/" + n;
+    struct stat st;
+    int r = ::stat(fn.c_str(), &st);
+    if (r == 0 && S_ISDIR(st.st_mode)) {
+      string initfn = fn + "/module.py";
+      r = ::stat(initfn.c_str(), &st);
+      if (r == 0) {
+       modules->insert(n);
+      }
+    }
+  }
+  closedir(dir);
+}
+
+void PyModules::list_modules(std::set<std::string> *modules)
+{
+  _list_modules(g_conf->mgr_module_path, modules);
+}

diff --git a/src/mgr/PyModules.h b/src/mgr/PyModules.h
index c98f7694bf64a94c2baf551995bc7577171be73b..93cbd5be8518a1a360a186ebe0b441a47a684ce5 100644 (file)
--- a/src/mgr/PyModules.h
+++ b/src/mgr/PyModules.h
@@ -101,6 +101,8 @@ public:
 
   void log(const std::string &handle,
            int level, const std::string &record);
+
+  static void list_modules(std::set<std::string> *modules);
 };
 
 #endif

diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h
index 4547bb4222f520504f260d3f49233970270128ba..3841840cff7a94644d0741ed5db26cf728e3059f 100644 (file)
--- a/src/mon/MgrMap.h
+++ b/src/mon/MgrMap.h
@@ -25,9 +25,11 @@ class StandbyInfo
 public:
   uint64_t gid;
   std::string name;
+  std::set<std::string> available_modules;
 
-  StandbyInfo(uint64_t gid_, const std::string &name_)
-    : gid(gid_), name(name_)
+  StandbyInfo(uint64_t gid_, const std::string &name_,
+             std::set<std::string>& am)
+    : gid(gid_), name(name_), available_modules(am)
   {}
 
   StandbyInfo()
@@ -36,17 +38,21 @@ public:
 
   void encode(bufferlist& bl) const
   {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(gid, bl);
     ::encode(name, bl);
+    ::encode(available_modules, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& p)
   {
-    DECODE_START(1, p);
+    DECODE_START(2, p);
     ::decode(gid, p);
     ::decode(name, p);
+    if (struct_v >= 2) {
+      ::decode(available_modules, p);
+    }
     DECODE_FINISH(p);
   }
 };
@@ -69,6 +75,7 @@ public:
   std::map<uint64_t, StandbyInfo> standbys;
 
   std::set<std::string> modules;
+  std::set<std::string> available_modules;
 
   epoch_t get_epoch() const { return epoch; }
   entity_addr_t get_active_addr() const { return active_addr; }
@@ -76,6 +83,18 @@ public:
   bool get_available() const { return available; }
   const std::string &get_active_name() const { return active_name; }
 
+  bool all_support_module(const std::string& module) {
+    if (!available_modules.count(module)) {
+      return false;
+    }
+    for (auto& p : standbys) {
+      if (!p.second.available_modules.count(module)) {
+       return false;
+      }
+    }
+    return true;
+  }
+
   void encode(bufferlist& bl, uint64_t features) const
   {
     ENCODE_START(2, 1, bl);
@@ -86,6 +105,7 @@ public:
     ::encode(active_name, bl);
     ::encode(standbys, bl);
     ::encode(modules, bl);
+    ::encode(available_modules, bl);
     ENCODE_FINISH(bl);
   }
 
@@ -100,6 +120,7 @@ public:
     ::decode(standbys, p);
     if (struct_v >= 2) {
       ::decode(modules, p);
+      ::decode(available_modules, p);
     }
     DECODE_FINISH(p);
   }
@@ -115,6 +136,11 @@ public:
       f->open_object_section("standby");
       f->dump_int("gid", i.second.gid);
       f->dump_string("name", i.second.name);
+      f->open_array_section("available_modules");
+      for (auto& j : i.second.available_modules) {
+       f->dump_string("module", j);
+      }
+      f->close_section();
       f->close_section();
     }
     f->close_section();
@@ -123,6 +149,11 @@ public:
       f->dump_string("module", i);
     }
     f->close_section();
+    f->open_array_section("available_modules");
+    for (auto& j : available_modules) {
+      f->dump_string("module", j);
+    }
+    f->close_section();
   }
 
   static void generate_test_instances(list<MgrMap*> &l) {

diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc
index 676cb92e9b10cb116c6217c0d095931797cf393c..dfdd0834ed3887ec13742cfd69988599e1b9ba35 100644 (file)
--- a/src/mon/MgrMonitor.cc
+++ b/src/mon/MgrMonitor.cc
@@ -213,6 +213,12 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
       pending_map.available = m->get_available();
       updated = true;
     }
+    if (pending_map.available_modules != m->get_available_modules()) {
+      dout(4) << "available_modules " << m->get_available_modules()
+             << " (was " << pending_map.available_modules << ")" << dendl;
+      pending_map.available_modules = m->get_available_modules();
+      updated = true;
+    }
   } else if (pending_map.active_gid == 0) {
     // There is no currently active daemon, select this one.
     if (pending_map.standbys.count(m->get_gid())) {
@@ -224,14 +230,26 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
            << pending_map.active_name << ")" << dendl;
     pending_map.active_gid = m->get_gid();
     pending_map.active_name = m->get_name();
+    pending_map.available_modules = m->get_available_modules();
 
     updated = true;
   } else {
     if (pending_map.standbys.count(m->get_gid()) > 0) {
       dout(10) << "from existing standby " << m->get_gid() << dendl;
+      if (pending_map.standbys[m->get_gid()].available_modules !=
+         m->get_available_modules()) {
+       dout(10) << "existing standby " << m->get_gid() << " available_modules "
+                << m->get_available_modules() << " (was "
+                << pending_map.standbys[m->get_gid()].available_modules << ")"
+                << dendl;
+       pending_map.standbys[m->get_gid()].available_modules =
+         m->get_available_modules();
+       updated = true;
+      }
     } else {
       dout(10) << "new standby " << m->get_gid() << dendl;
-      pending_map.standbys[m->get_gid()] = {m->get_gid(), m->get_name()};
+      pending_map.standbys[m->get_gid()] = {m->get_gid(), m->get_name(),
+                                           m->get_available_modules()};
       updated = true;
     }
   }
@@ -603,6 +621,15 @@ bool MgrMonitor::prepare_command(MonOpRequestRef op)
       r = -EINVAL;
       goto out;
     }
+    string force;
+    cmd_getval(g_ceph_context, cmdmap, "force", force);
+    if (!pending_map.all_support_module(module) &&
+       force != "--force") {
+      ss << "all mgr daemons do not support module '" << module << "', pass "
+        << "--force to force enablement";
+      r = -ENOENT;
+      goto out;
+    }
     pending_map.modules.insert(module);
   } else if (prefix == "mgr module disable") {
     string module;

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 0e5e308ab851e2a68e57afcefddfd7adcc37cbac..b45d2b7b0598c3a36bb3351e1f09fca4f5310f4b 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -952,9 +952,10 @@ COMMAND("mgr fail name=who,type=CephString", \
        "treat the named manager daemon as failed", "mgr", "rw", "cli,rest")
 COMMAND("mgr module ls",
        "list active mgr modules", "mgr", "r", "cli,rest")
-COMMAND("mgr module enable "                   \
-       "name=module,type=CephString",
+COMMAND("mgr module enable "                                           \
+       "name=module,type=CephString "                                  \
+       "name=force,type=CephChoices,strings=--force,req=false",
        "enable mgr module", "mgr", "rw", "cli,rest")
-COMMAND("mgr module disable "                  \
+COMMAND("mgr module disable "                                          \
        "name=module,type=CephString",
-       "enable mgr module", "mgr", "rw", "cli,rest")
+       "disable mgr module", "mgr", "rw", "cli,rest")