MDSMap: lock out multimds clusters and directory fragmentation by default

Add new flags operating the way the snapshot lockout does. Then, actually
block out the directory fragmentation in the MDS; and block out setting
max_mds > 1 in the MDSMonitor.

Signed-off-by: Greg Farnum <gfarnum@redhat.com>

diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index 593fd5f9793ed59fd931c92d1e25a076abc20a57..fe0a8d5d0699aa1ec50d1117ecdfe638e6764f58 100644 (file)
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -234,6 +234,11 @@ struct ceph_mon_subscribe_ack {
  */
 #define CEPH_MDSMAP_DOWN    (1<<0)  /* cluster deliberately down */
 #define CEPH_MDSMAP_ALLOW_SNAPS   (1<<1)  /* cluster allowed to create snapshots */
+#define CEPH_MDSMAP_ALLOW_MULTIMDS (1<<2) /* cluster allowed to have >1 active MDS */
+#define CEPH_MDSMAP_ALLOW_DIRFRAGS (1<<3) /* cluster allowed to fragment directories */
+
+#define CEPH_MDSMAP_ALLOW_CLASSICS (CEPH_MDSMAP_ALLOW_SNAPS | CEPH_MDSMAP_ALLOW_MULTIMDS | \
+                                   CEPH_MDSMAP_ALLOW_DIRFRAGS)
 
 /*
  * mds states

diff --git a/src/mds/FSMap.cc b/src/mds/FSMap.cc
index 92e9a1026cb4cbbcaac3af03d24edf6b1e3bcbcc..7b235fbe264cc549dbf894d1a66906cf6cedf2b1 100644 (file)
--- a/src/mds/FSMap.cc
+++ b/src/mds/FSMap.cc
@@ -345,16 +345,24 @@ void FSMap::decode(bufferlist::iterator& p)
        // previously this was a bool about snaps, not a flag map
        bool flag;
        ::decode(flag, p);
-       legacy_mds_map.ever_allowed_features = flag ? CEPH_MDSMAP_ALLOW_SNAPS : 0;
+       legacy_mds_map.ever_allowed_features = flag ?
+         CEPH_MDSMAP_ALLOW_SNAPS : 0;
        ::decode(flag, p);
-       legacy_mds_map.explicitly_allowed_features = flag ? CEPH_MDSMAP_ALLOW_SNAPS : 0;
+       legacy_mds_map.explicitly_allowed_features = flag ?
+         CEPH_MDSMAP_ALLOW_SNAPS : 0;
+       if (legacy_mds_map.max_mds > 1) {
+         legacy_mds_map.set_multimds_allowed();
+       }
       } else {
        ::decode(legacy_mds_map.ever_allowed_features, p);
        ::decode(legacy_mds_map.explicitly_allowed_features, p);
       }
     } else {
-      legacy_mds_map.ever_allowed_features = CEPH_MDSMAP_ALLOW_SNAPS;
+      legacy_mds_map.ever_allowed_features = CEPH_MDSMAP_ALLOW_CLASSICS;
       legacy_mds_map.explicitly_allowed_features = 0;
+      if (legacy_mds_map.max_mds > 1) {
+       legacy_mds_map.set_multimds_allowed();
+      }
     }
     if (ev >= 7)
       ::decode(legacy_mds_map.inline_data_enabled, p);

diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc
index 4c42568aa2607c36e99fb0c3dc35f5d5aea2e9f7..99845fb6cfcf56bd21f93ee7baf97afdadf769c5 100644 (file)
--- a/src/mds/MDBalancer.cc
+++ b/src/mds/MDBalancer.cc
@@ -333,6 +333,7 @@ double MDBalancer::try_match(mds_rank_t ex, double& maxex,
 
 void MDBalancer::queue_split(CDir *dir)
 {
+  assert(mds->mdsmap->allows_dirfrags());
   split_queue.insert(dir->dirfrag());
 }
 
@@ -984,6 +985,7 @@ void MDBalancer::hit_dir(utime_t now, CDir *dir, int type, int who, double amoun
 
     // split
     if (g_conf->mds_bal_split_size > 0 &&
+       mds->mdsmap->allows_dirfrags() &&
        (dir->should_split() ||
         (v > g_conf->mds_bal_split_rd && type == META_POP_IRD) ||
         (v > g_conf->mds_bal_split_wr && type == META_POP_IWR)) &&

diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc
index 16b6b2d4f99e2f555aa4547fff0844d7c581452a..f66fc7a85e4450aca94768923678fb96abef398a 100644 (file)
--- a/src/mds/MDSMap.cc
+++ b/src/mds/MDSMap.cc
@@ -630,16 +630,23 @@ void MDSMap::decode(bufferlist::iterator& p)
       // previously this was a bool about snaps, not a flag map
       bool flag;
       ::decode(flag, p);
-      legacy_mds_map.ever_allowed_features = flag ? CEPH_MDSMAP_ALLOW_SNAPS : 0;
+      ever_allowed_features = flag ? CEPH_MDSMAP_ALLOW_SNAPS : 0;
+      ever_allowed_features |= CEPH_MDSMAP_ALLOW_MULTIMDS|CEPH_MDSMAP_ALLOW_DIRFRAGS;
       ::decode(flag, p);
-      legacy_mds_map.explicitly_allowed_features = flag ? CEPH_MDSMAP_ALLOW_SNAPS : 0;
+      explicitly_allowed_features = flag ? CEPH_MDSMAP_ALLOW_SNAPS : 0;
+      if (max_mds > 1) {
+       set_multimds_allowed();
+      }
     } else {
       ::decode(ever_allowed_features, p);
       ::decode(explicitly_allowed_features, p);
     }
   } else {
-    ever_allowed_features = CEPH_MDSMAP_ALLOW_SNAPS;
+    ever_allowed_features = CEPH_MDSMAP_ALLOW_CLASSICS;
     explicitly_allowed_features = 0;
+    if (max_mds > 1) {
+      set_multimds_allowed();
+    }
   }
   if (ev >= 7)
     ::decode(inline_data_enabled, p);

diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h
index 597270e08e159d8ba72d819a6a76f93ca4df0eaf..bb69a75f9d5efa97c11a40aa0d50bcaec624ed3e 100644 (file)
--- a/src/mds/MDSMap.h
+++ b/src/mds/MDSMap.h
@@ -262,8 +262,24 @@ public:
     ever_allowed_features |= CEPH_MDSMAP_ALLOW_SNAPS;
     explicitly_allowed_features |= CEPH_MDSMAP_ALLOW_SNAPS;
   }
-  bool allows_snaps() { return test_flag(CEPH_MDSMAP_ALLOW_SNAPS); }
   void clear_snaps_allowed() { clear_flag(CEPH_MDSMAP_ALLOW_SNAPS); }
+  bool allows_snaps() const { return test_flag(CEPH_MDSMAP_ALLOW_SNAPS); }
+
+  void set_multimds_allowed() {
+    set_flag(CEPH_MDSMAP_ALLOW_MULTIMDS);
+    ever_allowed_features |= CEPH_MDSMAP_ALLOW_MULTIMDS;
+    explicitly_allowed_features |= CEPH_MDSMAP_ALLOW_MULTIMDS;
+  }
+  void clear_multimds_allowed() { clear_flag(CEPH_MDSMAP_ALLOW_MULTIMDS); }
+  bool allows_multimds() const { return test_flag(CEPH_MDSMAP_ALLOW_MULTIMDS); }
+
+  void set_dirfrags_allowed() {
+    set_flag(CEPH_MDSMAP_ALLOW_DIRFRAGS);
+    ever_allowed_features |= CEPH_MDSMAP_ALLOW_DIRFRAGS;
+    explicitly_allowed_features |= CEPH_MDSMAP_ALLOW_DIRFRAGS;
+  }
+  void clear_dirfrags_allowed() { clear_flag(CEPH_MDSMAP_ALLOW_DIRFRAGS); }
+  bool allows_dirfrags() const { return test_flag(CEPH_MDSMAP_ALLOW_DIRFRAGS); }
 
   epoch_t get_epoch() const { return epoch; }
   void inc_epoch() { epoch++; }

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 71e49258434f3c09dfb5dfcae6077530fd3bf85b..feb48970e1fdf0c3de43ae7bf72d5da8d57efd4a 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -502,7 +502,8 @@ bool MDSRank::_dispatch(Message *m, bool new_msg)
     if (!dir->get_parent_dir()) continue;    // must be linked.
     if (!dir->is_auth()) continue;           // must be auth.
     frag_t fg = dir->get_frag();
-    if (fg == frag_t() || (rand() % (1 << fg.bits()) == 0))
+    if (mdsmap->allows_dirfrags() &&
+       (fg == frag_t() || (rand() % (1 << fg.bits()) == 0)))
       mdcache->split_dir(dir, 1);
     else
       balancer->queue_merge(dir);
@@ -2166,6 +2167,11 @@ bool MDSRank::command_dirfrag_split(
     cmdmap_t cmdmap,
     std::ostream &ss)
 {
+  if (!mdsmap->allows_dirfrags()) {
+    ss << "dirfrags are disallowed by the mds map!";
+    return false;
+  }
+
   int64_t by = 0;
   if (!cmd_getval(g_ceph_context, cmdmap, "bits", by)) {
     ss << "missing bits argument";
@@ -2467,25 +2473,27 @@ bool MDSRankDispatcher::handle_command_legacy(std::vector<std::string> args)
       dout(20) << "try_eval(" << inum << ", " << mask << ")" << dendl;
     } else dout(15) << "inode " << inum << " not in mdcache!" << dendl;
   } else if (args[0] == "fragment_dir") {
-    if (args.size() == 4) {
-      filepath fp(args[1].c_str());
-      CInode *in = mdcache->cache_traverse(fp);
-      if (in) {
-       frag_t fg;
-       if (fg.parse(args[2].c_str())) {
-         CDir *dir = in->get_dirfrag(fg);
-         if (dir) {
-           if (dir->is_auth()) {
-             int by = atoi(args[3].c_str());
-             if (by)
-               mdcache->split_dir(dir, by);
-             else
-               dout(0) << "need to split by >0 bits" << dendl;
-           } else dout(0) << "dir " << dir->dirfrag() << " not auth" << dendl;
-         } else dout(0) << "dir " << in->ino() << " " << fg << " dne" << dendl;
-       } else dout(0) << " frag " << args[2] << " does not parse" << dendl;
-      } else dout(0) << "path " << fp << " not found" << dendl;
-    } else dout(0) << "bad syntax" << dendl;
+    if (!mdsmap->allows_dirfrags()) {
+      if (args.size() == 4) {
+       filepath fp(args[1].c_str());
+       CInode *in = mdcache->cache_traverse(fp);
+       if (in) {
+         frag_t fg;
+         if (fg.parse(args[2].c_str())) {
+           CDir *dir = in->get_dirfrag(fg);
+           if (dir) {
+             if (dir->is_auth()) {
+               int by = atoi(args[3].c_str());
+               if (by)
+                 mdcache->split_dir(dir, by);
+               else
+                 dout(0) << "need to split by >0 bits" << dendl;
+             } else dout(0) << "dir " << dir->dirfrag() << " not auth" << dendl;
+           } else dout(0) << "dir " << in->ino() << " " << fg << " dne" << dendl;
+         } else dout(0) << " frag " << args[2] << " does not parse" << dendl;
+       } else dout(0) << "path " << fp << " not found" << dendl;
+      } else dout(0) << "bad syntax" << dendl;
+    } else dout(0) << "dirfrags are disallowed by the mds map!" << dendl;
   } else if (args[0] == "merge_dir") {
     if (args.size() == 3) {
       filepath fp(args[1].c_str());

diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
index 7065ca0de7acc65dbf1c35cdf9de8c375d60f56c..78fddc6fa600415607b0549c95ef8c0dfa1f949b 100644 (file)
--- a/src/mon/MDSMonitor.cc
+++ b/src/mon/MDSMonitor.cc
@@ -1820,6 +1820,11 @@ public:
       if (interr.length()) {
        return -EINVAL;
       }
+      if (!fs->mds_map.allows_multimds() && n > fs->mds_map.get_max_mds() &&
+         n > 1) {
+       ss << "multi-MDS clusters are not enabled; set 'allow_multimds' to enable";
+       return -EINVAL;
+      }
       if (n > MAX_MDS) {
         ss << "may not have more than " << MAX_MDS << " MDS ranks";
         return -EINVAL;
@@ -1911,6 +1916,64 @@ public:
         });
        ss << "enabled new snapshots";
       }
+    } else if (var == "allow_multimds") {
+      bool enable_multimds = false;
+      int r = parse_bool(val, &enable_multimds, ss);
+      if (r != 0) {
+       return r;
+      }
+
+      if (!enable_multimds) {
+       fsmap.modify_filesystem(fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+               {
+                 fs->mds_map.clear_multimds_allowed();
+               });
+       ss << "disallowed increasing the cluster size past 1";
+      } else {
+       string confirm;
+       if (!cmd_getval(g_ceph_context, cmdmap, "confirm", confirm) ||
+           confirm != "--yes-i-really-mean-it") {
+         ss << "Multi-MDS clusters are unstable and will probably break your FS! Set to --yes-i-really-mean-it if you are sure you want to enable this";
+         return -EPERM;
+       }
+        fsmap.modify_filesystem(
+            fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+        {
+          fs->mds_map.set_multimds_allowed();
+        });
+       ss << "enabled creation of more than 1 active MDS";
+      }
+    } else if (var == "allow_dirfrags") {
+      bool enable_dirfrags = false;
+      int r = parse_bool(val, &enable_dirfrags, ss);
+      if (r != 0) {
+       return r;
+      }
+
+      if (!enable_dirfrags) {
+       fsmap.modify_filesystem(fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+               {
+                 fs->mds_map.clear_dirfrags_allowed();
+               });
+       ss << "disallowed new directory fragmentation";
+      } else {
+       string confirm;
+       if (!cmd_getval(g_ceph_context, cmdmap, "confirm", confirm) ||
+           confirm != "--yes-i-really-mean-it") {
+         ss << "directory fragmentation is unstable and may break your FS! Set to --yes-i-really-mean-it if you are sure you want to enable this";
+         return -EPERM;
+       }
+        fsmap.modify_filesystem(
+            fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+        {
+          fs->mds_map.set_dirfrags_allowed();
+        });
+       ss << "enabled directory fragmentation";
+      }
     } else if (var == "cluster_down") {
       bool is_down = false;
       int r = parse_bool(val, &is_down, ss);
@@ -2337,6 +2400,17 @@ int MDSMonitor::legacy_filesystem_command(
     if (!cmd_getval(g_ceph_context, cmdmap, "maxmds", maxmds) || maxmds < 0) {
       return -EINVAL;
     }
+
+    const MDSMap& mdsmap =
+      pending_fsmap.filesystems.at(pending_fsmap.legacy_client_fscid)->mds_map;
+      
+    if (!mdsmap.allows_multimds() &&
+       maxmds > mdsmap.get_max_mds() &&
+       maxmds > 1) {
+      ss << "multi-MDS clusters are not enabled; set 'allow_multimds' to enable";
+      return -EINVAL;
+    }
+
     if (maxmds > MAX_MDS) {
       ss << "may not have more than " << MAX_MDS << " MDS ranks";
       return -EINVAL;