From 048e825484cc0e633956e8e53e5fa24c3b23feca Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 15 Mar 2017 19:48:25 +0000 Subject: [PATCH] mds: make blacklisting configurable in kill_session This makes the code a bit awkward because we now have 4x paths through this function (blocking, noblocking, blacklisting, nonblacklisting). Signed-off-by: John Spray --- doc/cephfs/eviction.rst | 12 +++- src/common/config_opts.h | 4 ++ src/mds/MDSDaemon.cc | 3 +- src/mds/MDSRank.cc | 130 +++++++++++++++++++++++---------------- src/mds/MDSRank.h | 4 +- 5 files changed, 95 insertions(+), 58 deletions(-) diff --git a/doc/cephfs/eviction.rst b/doc/cephfs/eviction.rst index bd201cebbf7..a881dc6af4c 100644 --- a/doc/cephfs/eviction.rst +++ b/doc/cephfs/eviction.rst @@ -80,8 +80,16 @@ output: That's it! The client has now been evicted, and any resources it had locked will now be available for other clients. -Background: OSD epoch barrier ------------------------------ +Background: blacklisting and OSD epoch barrier +---------------------------------------------- + +After a client is blacklisted, it is necessary to make sure that +other clients and MDS daemons have the latest OSDMap (including +the blacklist entry) before they try to access any data objects +that the blacklisted client might have been accessing. + +This is ensured using an internal "osdmap epoch barrier" mechanism. +See :doc: The purpose of the barrier is to ensure that when we hand out any capabilities which might allow touching the same RADOS objects, the diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 13e1a558441..9b1efc857f7 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -511,7 +511,11 @@ OPTION(mds_beacon_interval, OPT_FLOAT, 4) OPTION(mds_beacon_grace, OPT_FLOAT, 15) OPTION(mds_enforce_unique_name, OPT_BOOL, true) OPTION(mds_blacklist_interval, OPT_FLOAT, 24.0*60.0) // how long to blacklist failed nodes + OPTION(mds_session_timeout, OPT_FLOAT, 60) // cap bits and leases time out if client idle +OPTION(mds_session_blacklist_on_timeout, OPT_BOOL, true) // whether to blacklist clients whose sessions are dropped due to timeout +OPTION(mds_session_blacklist_on_evict, OPT_BOOL, true) // whether to blacklist clients whose sessions are dropped via admin commands + OPTION(mds_sessionmap_keys_per_op, OPT_U32, 1024) // how many sessions should I try to load/store in a single OMAP operation? OPTION(mds_revoke_cap_timeout, OPT_FLOAT, 60) // detect clients which aren't revoking caps OPTION(mds_recall_state_timeout, OPT_FLOAT, 60) // detect clients which aren't trimming caps diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc index 4c3d558ed46..48c1b46551b 100644 --- a/src/mds/MDSDaemon.cc +++ b/src/mds/MDSDaemon.cc @@ -774,7 +774,8 @@ int MDSDaemon::_handle_command( int64_t session_id = 0; bool got = cmd_getval(cct, cmdmap, "session_id", session_id); assert(got); - bool killed = mds_rank->kill_session(session_id, false, ss); + bool killed = mds_rank->kill_session(session_id, false, + g_conf->mds_session_blacklist_on_evict, ss); if (!killed) r = -ENOENT; } else if (prefix == "heap") { diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index fe2750301df..2791f9487ef 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -1876,8 +1876,9 @@ bool MDSRankDispatcher::handle_asok_command( } mds_lock.Lock(); - stringstream dss; - bool killed = kill_session(strtol(client_id.c_str(), 0, 10), true, dss); + std::stringstream dss; + bool killed = kill_session(strtol(client_id.c_str(), 0, 10), true, + g_conf->mds_session_blacklist_on_evict, dss); if (!killed) { dout(15) << dss.str() << dendl; ss << dss.str(); @@ -2005,7 +2006,9 @@ void MDSRankDispatcher::evict_sessions(const SessionFilter &filter, MCommand *m) C_GatherBuilder gather(g_ceph_context, reply); for (const auto s : victims) { - server->kill_session(s, gather.new_sub()); + std::stringstream ss; + kill_session(s->info.inst.name.num(), false, + g_conf->mds_session_blacklist_on_evict, ss, gather.new_sub()); } gather.activate(); } @@ -2633,8 +2636,15 @@ void MDSRankDispatcher::handle_osd_map() objecter->maybe_request_map(); } -bool MDSRankDispatcher::kill_session(int64_t session_id, bool wait, std::stringstream& err_ss) +bool MDSRank::kill_session(int64_t session_id, + bool wait, bool blacklist, std::stringstream& err_ss, + Context *on_killed) { + assert(mds_lock.is_locked_by_me()); + + // Mutually exclusive args + assert(!(wait && on_killed != nullptr)); + if (is_any_replay()) { err_ss << "MDS is replaying log"; return false; @@ -2656,56 +2666,68 @@ bool MDSRankDispatcher::kill_session(int64_t session_id, bool wait, std::strings std::string tmp = ss.str(); std::vector cmd = {tmp}; - C_SaferCond on_blacklist_inline; - Context *on_blacklist_done = nullptr; - if (wait) { - on_blacklist_done = &on_blacklist_inline; - } else { - on_blacklist_done = new FunctionContext([this, session_id](int r) { - objecter->wait_for_latest_osdmap( - new FunctionContext([this, session_id](int r) { - auto epoch = objecter->with_osdmap([](const OSDMap &o){ - return o.get_epoch(); - }); - - set_osd_epoch_barrier(epoch); - - Session *session = sessionmap.get_session( - entity_name_t(CEPH_ENTITY_TYPE_CLIENT, session_id)); - if (session) { - server->kill_session(session, NULL); - } else { - dout(1) << "session " << session_id << " was removed while we waited " - "for blacklist" << dendl; - } - })); - }); - } + auto kill_mds_session = [this, session_id, on_killed](){ + assert(mds_lock.is_locked_by_me()); + Session *session = sessionmap.get_session( + entity_name_t(CEPH_ENTITY_TYPE_CLIENT, session_id)); + if (session) { + if (on_killed) { + server->kill_session(session, on_killed); + } else { + C_SaferCond on_safe; + server->kill_session(session, &on_safe); - dout(4) << "Sending mon blacklist command: " << cmd[0] << dendl; - monc->start_mon_command(cmd, {}, nullptr, nullptr, on_blacklist_done); + mds_lock.Unlock(); + on_safe.wait(); + mds_lock.Lock(); + } + } else { + dout(1) << "session " << session_id << " was removed while we waited " + "for blacklist" << dendl; - if (wait) { - mds_lock.Unlock(); - int r = on_blacklist_inline.wait(); - mds_lock.Lock(); - if (r != 0) { - err_ss << "Failed to blacklist, r=" << r; - return false; + // Even though it wasn't us that removed it, kick our completion + // as the session has been removed. + if (on_killed) { + on_killed->complete(0); + } } + }; + + auto background_blacklist = [this, session_id, cmd](std::function fn){ + assert(mds_lock.is_locked_by_me()); + + Context *on_blacklist_done = new FunctionContext([this, session_id, fn](int r) { + objecter->wait_for_latest_osdmap( + new C_OnFinisher( + new FunctionContext([this, session_id, fn](int r) { + Mutex::Locker l(mds_lock); + auto epoch = objecter->with_osdmap([](const OSDMap &o){ + return o.get_epoch(); + }); + + set_osd_epoch_barrier(epoch); + + fn(); + }), finisher) + ); + }); + + dout(4) << "Sending mon blacklist command: " << cmd[0] << dendl; + monc->start_mon_command(cmd, {}, nullptr, nullptr, on_blacklist_done); + }; - // Wait for latest OSDMap so that we can learn the OSDMap epoch in - // which the blacklist happened - C_SaferCond on_latest_osdmap; - objecter->wait_for_latest_osdmap(&on_latest_osdmap); + auto blocking_blacklist = [this, cmd, &err_ss, background_blacklist](){ + C_SaferCond inline_ctx; + background_blacklist([&inline_ctx](){inline_ctx.complete(0);}); mds_lock.Unlock(); - on_latest_osdmap.wait(); + inline_ctx.wait(); mds_lock.Lock(); - auto epoch = objecter->with_osdmap([](const OSDMap &o){ - return o.get_epoch(); - }); + }; - set_osd_epoch_barrier(epoch); + if (wait) { + if (blacklist) { + blocking_blacklist(); + } // We dropped mds_lock, so check that session still exists session = sessionmap.get_session(entity_name_t(CEPH_ENTITY_TYPE_CLIENT, @@ -2715,13 +2737,13 @@ bool MDSRankDispatcher::kill_session(int64_t session_id, bool wait, std::strings "for blacklist" << dendl; return true; } - - C_SaferCond on_safe; - server->kill_session(session, &on_safe); - - mds_lock.Unlock(); - on_safe.wait(); - mds_lock.Lock(); + kill_mds_session(); + } else { + if (blacklist) { + background_blacklist(kill_mds_session); + } else { + kill_mds_session(); + } } return true; diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index af46f4c2f3e..5fae1b43e9f 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -406,6 +406,9 @@ class MDSRank { return map_targets.count(rank); } + bool kill_session(int64_t session_id, bool wait, bool blacklist, + std::stringstream& ss, Context *on_killed=nullptr); + protected: void dump_clientreplay_status(Formatter *f) const; void command_scrub_path(Formatter *f, const string& path, vector& scrubop_vec); @@ -535,7 +538,6 @@ public: Formatter *f, std::ostream& ss); void handle_mds_map(MMDSMap *m, MDSMap *oldmap); void handle_osd_map(); - bool kill_session(int64_t session_id, bool wait, std::stringstream& ss); void update_log_config(); bool handle_command( -- 2.39.5