From 4bc4e74932b6f07bb41d944318719c7bd5700634 Mon Sep 17 00:00:00 2001 From: Greg Farnum Date: Wed, 30 Mar 2016 19:32:55 -0700 Subject: [PATCH] MDSMap: lock out multimds clusters and directory fragmentation by default Add new flags operating the way the snapshot lockout does. Then, actually block out the directory fragmentation in the MDS; and block out setting max_mds > 1 in the MDSMonitor. Signed-off-by: Greg Farnum --- src/include/ceph_fs.h | 5 +++ src/mds/FSMap.cc | 14 ++++++-- src/mds/MDBalancer.cc | 2 ++ src/mds/MDSMap.cc | 13 ++++++-- src/mds/MDSMap.h | 18 ++++++++++- src/mds/MDSRank.cc | 48 ++++++++++++++++------------ src/mon/MDSMonitor.cc | 74 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 147 insertions(+), 27 deletions(-) diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h index 593fd5f9793..fe0a8d5d069 100644 --- a/src/include/ceph_fs.h +++ b/src/include/ceph_fs.h @@ -234,6 +234,11 @@ struct ceph_mon_subscribe_ack { */ #define CEPH_MDSMAP_DOWN (1<<0) /* cluster deliberately down */ #define CEPH_MDSMAP_ALLOW_SNAPS (1<<1) /* cluster allowed to create snapshots */ +#define CEPH_MDSMAP_ALLOW_MULTIMDS (1<<2) /* cluster allowed to have >1 active MDS */ +#define CEPH_MDSMAP_ALLOW_DIRFRAGS (1<<3) /* cluster allowed to fragment directories */ + +#define CEPH_MDSMAP_ALLOW_CLASSICS (CEPH_MDSMAP_ALLOW_SNAPS | CEPH_MDSMAP_ALLOW_MULTIMDS | \ + CEPH_MDSMAP_ALLOW_DIRFRAGS) /* * mds states diff --git a/src/mds/FSMap.cc b/src/mds/FSMap.cc index 92e9a1026cb..7b235fbe264 100644 --- a/src/mds/FSMap.cc +++ b/src/mds/FSMap.cc @@ -345,16 +345,24 @@ void FSMap::decode(bufferlist::iterator& p) // previously this was a bool about snaps, not a flag map bool flag; ::decode(flag, p); - legacy_mds_map.ever_allowed_features = flag ? CEPH_MDSMAP_ALLOW_SNAPS : 0; + legacy_mds_map.ever_allowed_features = flag ? + CEPH_MDSMAP_ALLOW_SNAPS : 0; ::decode(flag, p); - legacy_mds_map.explicitly_allowed_features = flag ? CEPH_MDSMAP_ALLOW_SNAPS : 0; + legacy_mds_map.explicitly_allowed_features = flag ? + CEPH_MDSMAP_ALLOW_SNAPS : 0; + if (legacy_mds_map.max_mds > 1) { + legacy_mds_map.set_multimds_allowed(); + } } else { ::decode(legacy_mds_map.ever_allowed_features, p); ::decode(legacy_mds_map.explicitly_allowed_features, p); } } else { - legacy_mds_map.ever_allowed_features = CEPH_MDSMAP_ALLOW_SNAPS; + legacy_mds_map.ever_allowed_features = CEPH_MDSMAP_ALLOW_CLASSICS; legacy_mds_map.explicitly_allowed_features = 0; + if (legacy_mds_map.max_mds > 1) { + legacy_mds_map.set_multimds_allowed(); + } } if (ev >= 7) ::decode(legacy_mds_map.inline_data_enabled, p); diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc index 4c42568aa26..99845fb6cfc 100644 --- a/src/mds/MDBalancer.cc +++ b/src/mds/MDBalancer.cc @@ -333,6 +333,7 @@ double MDBalancer::try_match(mds_rank_t ex, double& maxex, void MDBalancer::queue_split(CDir *dir) { + assert(mds->mdsmap->allows_dirfrags()); split_queue.insert(dir->dirfrag()); } @@ -984,6 +985,7 @@ void MDBalancer::hit_dir(utime_t now, CDir *dir, int type, int who, double amoun // split if (g_conf->mds_bal_split_size > 0 && + mds->mdsmap->allows_dirfrags() && (dir->should_split() || (v > g_conf->mds_bal_split_rd && type == META_POP_IRD) || (v > g_conf->mds_bal_split_wr && type == META_POP_IWR)) && diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc index 16b6b2d4f99..f66fc7a85e4 100644 --- a/src/mds/MDSMap.cc +++ b/src/mds/MDSMap.cc @@ -630,16 +630,23 @@ void MDSMap::decode(bufferlist::iterator& p) // previously this was a bool about snaps, not a flag map bool flag; ::decode(flag, p); - legacy_mds_map.ever_allowed_features = flag ? CEPH_MDSMAP_ALLOW_SNAPS : 0; + ever_allowed_features = flag ? CEPH_MDSMAP_ALLOW_SNAPS : 0; + ever_allowed_features |= CEPH_MDSMAP_ALLOW_MULTIMDS|CEPH_MDSMAP_ALLOW_DIRFRAGS; ::decode(flag, p); - legacy_mds_map.explicitly_allowed_features = flag ? CEPH_MDSMAP_ALLOW_SNAPS : 0; + explicitly_allowed_features = flag ? CEPH_MDSMAP_ALLOW_SNAPS : 0; + if (max_mds > 1) { + set_multimds_allowed(); + } } else { ::decode(ever_allowed_features, p); ::decode(explicitly_allowed_features, p); } } else { - ever_allowed_features = CEPH_MDSMAP_ALLOW_SNAPS; + ever_allowed_features = CEPH_MDSMAP_ALLOW_CLASSICS; explicitly_allowed_features = 0; + if (max_mds > 1) { + set_multimds_allowed(); + } } if (ev >= 7) ::decode(inline_data_enabled, p); diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h index 597270e08e1..bb69a75f9d5 100644 --- a/src/mds/MDSMap.h +++ b/src/mds/MDSMap.h @@ -262,8 +262,24 @@ public: ever_allowed_features |= CEPH_MDSMAP_ALLOW_SNAPS; explicitly_allowed_features |= CEPH_MDSMAP_ALLOW_SNAPS; } - bool allows_snaps() { return test_flag(CEPH_MDSMAP_ALLOW_SNAPS); } void clear_snaps_allowed() { clear_flag(CEPH_MDSMAP_ALLOW_SNAPS); } + bool allows_snaps() const { return test_flag(CEPH_MDSMAP_ALLOW_SNAPS); } + + void set_multimds_allowed() { + set_flag(CEPH_MDSMAP_ALLOW_MULTIMDS); + ever_allowed_features |= CEPH_MDSMAP_ALLOW_MULTIMDS; + explicitly_allowed_features |= CEPH_MDSMAP_ALLOW_MULTIMDS; + } + void clear_multimds_allowed() { clear_flag(CEPH_MDSMAP_ALLOW_MULTIMDS); } + bool allows_multimds() const { return test_flag(CEPH_MDSMAP_ALLOW_MULTIMDS); } + + void set_dirfrags_allowed() { + set_flag(CEPH_MDSMAP_ALLOW_DIRFRAGS); + ever_allowed_features |= CEPH_MDSMAP_ALLOW_DIRFRAGS; + explicitly_allowed_features |= CEPH_MDSMAP_ALLOW_DIRFRAGS; + } + void clear_dirfrags_allowed() { clear_flag(CEPH_MDSMAP_ALLOW_DIRFRAGS); } + bool allows_dirfrags() const { return test_flag(CEPH_MDSMAP_ALLOW_DIRFRAGS); } epoch_t get_epoch() const { return epoch; } void inc_epoch() { epoch++; } diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index 71e49258434..feb48970e1f 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -502,7 +502,8 @@ bool MDSRank::_dispatch(Message *m, bool new_msg) if (!dir->get_parent_dir()) continue; // must be linked. if (!dir->is_auth()) continue; // must be auth. frag_t fg = dir->get_frag(); - if (fg == frag_t() || (rand() % (1 << fg.bits()) == 0)) + if (mdsmap->allows_dirfrags() && + (fg == frag_t() || (rand() % (1 << fg.bits()) == 0))) mdcache->split_dir(dir, 1); else balancer->queue_merge(dir); @@ -2166,6 +2167,11 @@ bool MDSRank::command_dirfrag_split( cmdmap_t cmdmap, std::ostream &ss) { + if (!mdsmap->allows_dirfrags()) { + ss << "dirfrags are disallowed by the mds map!"; + return false; + } + int64_t by = 0; if (!cmd_getval(g_ceph_context, cmdmap, "bits", by)) { ss << "missing bits argument"; @@ -2467,25 +2473,27 @@ bool MDSRankDispatcher::handle_command_legacy(std::vector args) dout(20) << "try_eval(" << inum << ", " << mask << ")" << dendl; } else dout(15) << "inode " << inum << " not in mdcache!" << dendl; } else if (args[0] == "fragment_dir") { - if (args.size() == 4) { - filepath fp(args[1].c_str()); - CInode *in = mdcache->cache_traverse(fp); - if (in) { - frag_t fg; - if (fg.parse(args[2].c_str())) { - CDir *dir = in->get_dirfrag(fg); - if (dir) { - if (dir->is_auth()) { - int by = atoi(args[3].c_str()); - if (by) - mdcache->split_dir(dir, by); - else - dout(0) << "need to split by >0 bits" << dendl; - } else dout(0) << "dir " << dir->dirfrag() << " not auth" << dendl; - } else dout(0) << "dir " << in->ino() << " " << fg << " dne" << dendl; - } else dout(0) << " frag " << args[2] << " does not parse" << dendl; - } else dout(0) << "path " << fp << " not found" << dendl; - } else dout(0) << "bad syntax" << dendl; + if (!mdsmap->allows_dirfrags()) { + if (args.size() == 4) { + filepath fp(args[1].c_str()); + CInode *in = mdcache->cache_traverse(fp); + if (in) { + frag_t fg; + if (fg.parse(args[2].c_str())) { + CDir *dir = in->get_dirfrag(fg); + if (dir) { + if (dir->is_auth()) { + int by = atoi(args[3].c_str()); + if (by) + mdcache->split_dir(dir, by); + else + dout(0) << "need to split by >0 bits" << dendl; + } else dout(0) << "dir " << dir->dirfrag() << " not auth" << dendl; + } else dout(0) << "dir " << in->ino() << " " << fg << " dne" << dendl; + } else dout(0) << " frag " << args[2] << " does not parse" << dendl; + } else dout(0) << "path " << fp << " not found" << dendl; + } else dout(0) << "bad syntax" << dendl; + } else dout(0) << "dirfrags are disallowed by the mds map!" << dendl; } else if (args[0] == "merge_dir") { if (args.size() == 3) { filepath fp(args[1].c_str()); diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc index 7065ca0de7a..78fddc6fa60 100644 --- a/src/mon/MDSMonitor.cc +++ b/src/mon/MDSMonitor.cc @@ -1820,6 +1820,11 @@ public: if (interr.length()) { return -EINVAL; } + if (!fs->mds_map.allows_multimds() && n > fs->mds_map.get_max_mds() && + n > 1) { + ss << "multi-MDS clusters are not enabled; set 'allow_multimds' to enable"; + return -EINVAL; + } if (n > MAX_MDS) { ss << "may not have more than " << MAX_MDS << " MDS ranks"; return -EINVAL; @@ -1911,6 +1916,64 @@ public: }); ss << "enabled new snapshots"; } + } else if (var == "allow_multimds") { + bool enable_multimds = false; + int r = parse_bool(val, &enable_multimds, ss); + if (r != 0) { + return r; + } + + if (!enable_multimds) { + fsmap.modify_filesystem(fs->fscid, + [](std::shared_ptr fs) + { + fs->mds_map.clear_multimds_allowed(); + }); + ss << "disallowed increasing the cluster size past 1"; + } else { + string confirm; + if (!cmd_getval(g_ceph_context, cmdmap, "confirm", confirm) || + confirm != "--yes-i-really-mean-it") { + ss << "Multi-MDS clusters are unstable and will probably break your FS! Set to --yes-i-really-mean-it if you are sure you want to enable this"; + return -EPERM; + } + fsmap.modify_filesystem( + fs->fscid, + [](std::shared_ptr fs) + { + fs->mds_map.set_multimds_allowed(); + }); + ss << "enabled creation of more than 1 active MDS"; + } + } else if (var == "allow_dirfrags") { + bool enable_dirfrags = false; + int r = parse_bool(val, &enable_dirfrags, ss); + if (r != 0) { + return r; + } + + if (!enable_dirfrags) { + fsmap.modify_filesystem(fs->fscid, + [](std::shared_ptr fs) + { + fs->mds_map.clear_dirfrags_allowed(); + }); + ss << "disallowed new directory fragmentation"; + } else { + string confirm; + if (!cmd_getval(g_ceph_context, cmdmap, "confirm", confirm) || + confirm != "--yes-i-really-mean-it") { + ss << "directory fragmentation is unstable and may break your FS! Set to --yes-i-really-mean-it if you are sure you want to enable this"; + return -EPERM; + } + fsmap.modify_filesystem( + fs->fscid, + [](std::shared_ptr fs) + { + fs->mds_map.set_dirfrags_allowed(); + }); + ss << "enabled directory fragmentation"; + } } else if (var == "cluster_down") { bool is_down = false; int r = parse_bool(val, &is_down, ss); @@ -2337,6 +2400,17 @@ int MDSMonitor::legacy_filesystem_command( if (!cmd_getval(g_ceph_context, cmdmap, "maxmds", maxmds) || maxmds < 0) { return -EINVAL; } + + const MDSMap& mdsmap = + pending_fsmap.filesystems.at(pending_fsmap.legacy_client_fscid)->mds_map; + + if (!mdsmap.allows_multimds() && + maxmds > mdsmap.get_max_mds() && + maxmds > 1) { + ss << "multi-MDS clusters are not enabled; set 'allow_multimds' to enable"; + return -EINVAL; + } + if (maxmds > MAX_MDS) { ss << "may not have more than " << MAX_MDS << " MDS ranks"; return -EINVAL; -- 2.39.5