From eeac2c76ba0dc4f23278b5a6d35105c542df876d Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Mon, 21 May 2018 17:11:45 +0800 Subject: [PATCH] mds: introduce "ceph fs set min_compat_client " The command set the oldest version of client that is allowed to connect to mds. Signed-off-by: "Yan, Zheng" --- src/common/ceph_strings.cc | 2 ++ src/mds/MDSMap.cc | 11 ++++++++- src/mds/MDSMap.h | 5 +++++ src/mds/MDSRank.cc | 3 +++ src/mds/Server.cc | 46 +++++++++++++++++++++++++++++--------- src/mds/Server.h | 7 ++++-- src/mds/cephfs_features.h | 7 +++++- src/mds/mdstypes.h | 4 ++++ src/mon/FSCommands.cc | 12 ++++++++++ src/mon/MonCommands.h | 2 +- 10 files changed, 83 insertions(+), 16 deletions(-) diff --git a/src/common/ceph_strings.cc b/src/common/ceph_strings.cc index df0b973474a02..a38dd690d84c0 100644 --- a/src/common/ceph_strings.cc +++ b/src/common/ceph_strings.cc @@ -92,6 +92,8 @@ const char *ceph_release_name(int r) case CEPH_RELEASE_NAUTILUS: return "nautilus"; default: + if (r < 0) + return "unspecified"; return "unknown"; } } diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc index 5e3f0b8d021bb..9f985da69530f 100644 --- a/src/mds/MDSMap.cc +++ b/src/mds/MDSMap.cc @@ -145,6 +145,8 @@ void MDSMap::dump(Formatter *f) const f->dump_int("root", root); f->dump_int("session_timeout", session_timeout); f->dump_int("session_autoclose", session_autoclose); + f->dump_stream("min_compat_client") << (int)min_compat_client << " (" + << ceph_release_name(min_compat_client) << ")"; f->dump_int("max_file_size", max_file_size); f->dump_int("last_failure", last_failure); f->dump_int("last_failure_osd_epoch", last_failure_osd_epoch); @@ -223,6 +225,8 @@ void MDSMap::print(ostream& out) const out << "session_timeout\t" << session_timeout << "\n" << "session_autoclose\t" << session_autoclose << "\n"; out << "max_file_size\t" << max_file_size << "\n"; + out << "min_compat_client\t" << (int)min_compat_client << " (" + << ceph_release_name(min_compat_client) << ")\n"; out << "last_failure\t" << last_failure << "\n" << "last_failure_osd_epoch\t" << last_failure_osd_epoch << "\n"; out << "compat\t" << compat << "\n"; @@ -682,7 +686,7 @@ void MDSMap::encode(bufferlist& bl, uint64_t features) const encode(cas_pool, bl); // kclient ignores everything from here - __u16 ev = 13; + __u16 ev = 14; encode(ev, bl); encode(compat, bl); encode(metadata_pool, bl); @@ -704,6 +708,7 @@ void MDSMap::encode(bufferlist& bl, uint64_t features) const encode(balancer, bl); encode(standby_count_wanted, bl); encode(old_max_mds, bl); + encode(min_compat_client, bl); ENCODE_FINISH(bl); } @@ -831,6 +836,10 @@ void MDSMap::decode(bufferlist::const_iterator& p) decode(old_max_mds, p); } + if (ev >= 14) { + decode(min_compat_client, p); + } + DECODE_FINISH(p); } diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h index afee8affc7661..c23e63498abd7 100644 --- a/src/mds/MDSMap.h +++ b/src/mds/MDSMap.h @@ -189,6 +189,8 @@ protected: __u32 session_autoclose = 300; uint64_t max_file_size = 1ULL<<40; /* 1TB */ + int8_t min_compat_client = -1; + std::vector data_pools; // file data pools available to clients (via an ioctl). first is the default. int64_t cas_pool = -1; // where CAS objects go int64_t metadata_pool = -1; // where fs metadata objects go @@ -249,6 +251,9 @@ public: uint64_t get_max_filesize() const { return max_file_size; } void set_max_filesize(uint64_t m) { max_file_size = m; } + + uint8_t get_min_compat_client() const { return min_compat_client; } + void set_min_compat_client(uint8_t version) { min_compat_client = version; } int get_flags() const { return flags; } bool test_flag(int f) const { return flags & f; } diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index a60f8ce4bfec3..054e247116573 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -1736,6 +1736,9 @@ void MDSRankDispatcher::handle_mds_map( if (objecter->get_client_incarnation() != incarnation) objecter->set_client_incarnation(incarnation); + if (oldmap->get_min_compat_client() != mdsmap->get_min_compat_client()) + server->update_required_client_features(); + // for debug if (g_conf()->mds_dump_cache_on_map) mdcache->dump_cache(); diff --git a/src/mds/Server.cc b/src/mds/Server.cc index 02cc8db601267..6c09976160d6e 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -379,7 +379,7 @@ void Server::handle_client_session(MClientSession *m) dout(20) << " " << p.first << ": " << p.second << dendl; } - feature_bitset_t missing_features(CEPHFS_FEATURES_MDS_REQUIRED); + feature_bitset_t missing_features = required_client_features; missing_features -= client_metadata.features; if (!missing_features.empty()) { stringstream ss; @@ -970,21 +970,35 @@ void Server::handle_client_reconnect(MClientReconnect *m) << ") from " << m->get_source_inst() << " after " << delay << " (allowed interval " << g_conf()->mds_reconnect_timeout << ")"; deny = true; - } else if (!session->is_open()) { - dout(1) << " session is closed, ignoring reconnect, sending close" << dendl; - mds->clog->info() << "denied reconnect attempt (mds is " - << ceph_mds_state_name(mds->get_state()) - << ") from " << m->get_source_inst() << " (session is closed)"; - deny = true; - } else if (mdcache->is_readonly()) { - dout(1) << " read-only FS, ignoring reconnect, sending close" << dendl; - mds->clog->info() << "denied reconnect attempt (mds is read-only)"; - deny = true; + } else { + std::string error_str; + if (!session->is_open()) { + error_str = "session is closed"; + } else if (mdcache->is_readonly()) { + error_str = "mds is readonly"; + } else { + feature_bitset_t missing_features = required_client_features; + missing_features -= session->info.client_metadata.features; + if (!missing_features.empty()) { + stringstream ss; + ss << "missing required features '" << missing_features << "'"; + error_str = ss.str(); + } + } + + if (!error_str.empty()) { + deny = true; + dout(1) << " " << error_str << ", ignoring reconnect, sending close" << dendl; + mds->clog->info() << "denied reconnect attempt from " + << m->get_source_inst() << " (" << error_str << ")"; + } } if (deny) { m->get_connection()->send_message(new MClientSession(CEPH_SESSION_CLOSE)); m->put(); + if (session->is_open()) + kill_session(session, nullptr); return; } @@ -1071,7 +1085,17 @@ void Server::handle_client_reconnect(MClientReconnect *m) m->put(); } +void Server::update_required_client_features() +{ + vector bits = CEPHFS_FEATURES_MDS_REQUIRED; + + int min_compat = mds->mdsmap->get_min_compat_client(); + if (min_compat >= CEPH_RELEASE_MIMIC) + bits.push_back(CEPHFS_FEATURE_MIMIC); + std::sort(bits.begin(), bits.end()); + required_client_features = feature_bitset_t(bits); +} void Server::reconnect_gather_finish() { diff --git a/src/mds/Server.h b/src/mds/Server.h index b468eb674aa3e..a23bb8c080835 100644 --- a/src/mds/Server.h +++ b/src/mds/Server.h @@ -84,8 +84,11 @@ private: int failed_reconnects; bool reconnect_evicting; // true if I am waiting for evictions to complete // before proceeding to reconnect_gather_finish + utime_t reconnect_start; + set client_reconnect_gather; // clients i need a reconnect msg from. feature_bitset_t supported_features; + feature_bitset_t required_client_features; friend class MDSContinuation; friend class ServerContext; @@ -109,8 +112,6 @@ public: void handle_osd_map(); // -- sessions and recovery -- - utime_t reconnect_start; - set client_reconnect_gather; // clients i need a reconnect msg from. bool waiting_for_reconnect(client_t c) const; void dump_reconnect_status(Formatter *f) const; @@ -131,6 +132,8 @@ public: void journal_close_session(Session *session, int state, Context *on_safe); void reconnect_clients(MDSInternalContext *reconnect_done_); void handle_client_reconnect(class MClientReconnect *m); + void update_required_client_features(); + //void process_reconnect_cap(CInode *in, int from, ceph_mds_cap_reconnect& capinfo); void reconnect_gather_finish(); void reconnect_tick(); diff --git a/src/mds/cephfs_features.h b/src/mds/cephfs_features.h index f1ad8a68b1f73..e141fd84dedd5 100644 --- a/src/mds/cephfs_features.h +++ b/src/mds/cephfs_features.h @@ -15,9 +15,14 @@ #ifndef CEPHFS_FEATURES_H #define CEPHFS_FEATURES_H -#define CEPHFS_FEATURE_MIMIC 0 +// Please add feature bits for later ceph releases and update +// Server::update_required_client_features(). + +// The first 8 bits are reserved for old ceph releases. +#define CEPHFS_FEATURE_MIMIC 8 #define CEPHFS_FEATURES_ALL { \ + 0, 1, 2, 3, 4, 5, 6, 7, \ CEPHFS_FEATURE_MIMIC, \ } diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h index 6569dfe9274b7..4453c46f5d913 100644 --- a/src/mds/mdstypes.h +++ b/src/mds/mdstypes.h @@ -1075,6 +1075,10 @@ public: _vec = other._vec; return *this; } + feature_bitset_t& operator=(feature_bitset_t&& other) { + _vec = std::move(other._vec); + return *this; + } bool empty() const { for (auto& v : _vec) { if (v) diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc index 992c712608b68..d5358216240aa 100644 --- a/src/mon/FSCommands.cc +++ b/src/mon/FSCommands.cc @@ -529,6 +529,18 @@ public: { fs->mds_map.set_session_autoclose((uint32_t)n); }); + } else if (var == "min_compat_client") { + int vno = ceph_release_from_name(val.c_str()); + if (vno <= 0) { + ss << "version " << val << " is not recognized"; + return -EINVAL; + } + fsmap.modify_filesystem( + fs->fscid, + [vno](std::shared_ptr fs) + { + fs->mds_map.set_min_compat_client((uint8_t)vno); + }); } else { ss << "unknown variable " << var; return -EINVAL; diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index eb9e55cba8e5b..58b676717f929 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -391,7 +391,7 @@ COMMAND("fs set " \ "name=var,type=CephChoices,strings=max_mds|max_file_size" "|allow_new_snaps|inline_data|cluster_down|allow_dirfrags|balancer" \ "|standby_count_wanted|session_timeout|session_autoclose" \ - "|down|joinable " \ + "|down|joinable|min_compat_client " \ "name=val,type=CephString " \ "name=confirm,type=CephString,req=false", \ "set fs parameter to ", "mds", "rw", "cli,rest") -- 2.39.5