From: John Spray Date: Wed, 15 Mar 2017 19:48:25 +0000 (+0000) Subject: mds: make blacklisting configurable in kill_session X-Git-Tag: ses5-milestone6~8^2~9^2~49^2~9 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=048e825484cc0e633956e8e53e5fa24c3b23feca;p=ceph.git mds: make blacklisting configurable in kill_session This makes the code a bit awkward because we now have 4x paths through this function (blocking, noblocking, blacklisting, nonblacklisting). Signed-off-by: John Spray --- diff --git a/doc/cephfs/eviction.rst b/doc/cephfs/eviction.rst index bd201cebbf76b..a881dc6af4c1c 100644 --- a/doc/cephfs/eviction.rst +++ b/doc/cephfs/eviction.rst @@ -80,8 +80,16 @@ output: That's it! The client has now been evicted, and any resources it had locked will now be available for other clients. -Background: OSD epoch barrier ------------------------------ +Background: blacklisting and OSD epoch barrier +---------------------------------------------- + +After a client is blacklisted, it is necessary to make sure that +other clients and MDS daemons have the latest OSDMap (including +the blacklist entry) before they try to access any data objects +that the blacklisted client might have been accessing. + +This is ensured using an internal "osdmap epoch barrier" mechanism. +See :doc: The purpose of the barrier is to ensure that when we hand out any capabilities which might allow touching the same RADOS objects, the diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 13e1a558441ce..9b1efc857f735 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -511,7 +511,11 @@ OPTION(mds_beacon_interval, OPT_FLOAT, 4) OPTION(mds_beacon_grace, OPT_FLOAT, 15) OPTION(mds_enforce_unique_name, OPT_BOOL, true) OPTION(mds_blacklist_interval, OPT_FLOAT, 24.0*60.0) // how long to blacklist failed nodes + OPTION(mds_session_timeout, OPT_FLOAT, 60) // cap bits and leases time out if client idle +OPTION(mds_session_blacklist_on_timeout, OPT_BOOL, true) // whether to blacklist clients whose sessions are dropped due to timeout +OPTION(mds_session_blacklist_on_evict, OPT_BOOL, true) // whether to blacklist clients whose sessions are dropped via admin commands + OPTION(mds_sessionmap_keys_per_op, OPT_U32, 1024) // how many sessions should I try to load/store in a single OMAP operation? OPTION(mds_revoke_cap_timeout, OPT_FLOAT, 60) // detect clients which aren't revoking caps OPTION(mds_recall_state_timeout, OPT_FLOAT, 60) // detect clients which aren't trimming caps diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc index 4c3d558ed46b9..48c1b46551b71 100644 --- a/src/mds/MDSDaemon.cc +++ b/src/mds/MDSDaemon.cc @@ -774,7 +774,8 @@ int MDSDaemon::_handle_command( int64_t session_id = 0; bool got = cmd_getval(cct, cmdmap, "session_id", session_id); assert(got); - bool killed = mds_rank->kill_session(session_id, false, ss); + bool killed = mds_rank->kill_session(session_id, false, + g_conf->mds_session_blacklist_on_evict, ss); if (!killed) r = -ENOENT; } else if (prefix == "heap") { diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index fe2750301dfb4..2791f9487eff4 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -1876,8 +1876,9 @@ bool MDSRankDispatcher::handle_asok_command( } mds_lock.Lock(); - stringstream dss; - bool killed = kill_session(strtol(client_id.c_str(), 0, 10), true, dss); + std::stringstream dss; + bool killed = kill_session(strtol(client_id.c_str(), 0, 10), true, + g_conf->mds_session_blacklist_on_evict, dss); if (!killed) { dout(15) << dss.str() << dendl; ss << dss.str(); @@ -2005,7 +2006,9 @@ void MDSRankDispatcher::evict_sessions(const SessionFilter &filter, MCommand *m) C_GatherBuilder gather(g_ceph_context, reply); for (const auto s : victims) { - server->kill_session(s, gather.new_sub()); + std::stringstream ss; + kill_session(s->info.inst.name.num(), false, + g_conf->mds_session_blacklist_on_evict, ss, gather.new_sub()); } gather.activate(); } @@ -2633,8 +2636,15 @@ void MDSRankDispatcher::handle_osd_map() objecter->maybe_request_map(); } -bool MDSRankDispatcher::kill_session(int64_t session_id, bool wait, std::stringstream& err_ss) +bool MDSRank::kill_session(int64_t session_id, + bool wait, bool blacklist, std::stringstream& err_ss, + Context *on_killed) { + assert(mds_lock.is_locked_by_me()); + + // Mutually exclusive args + assert(!(wait && on_killed != nullptr)); + if (is_any_replay()) { err_ss << "MDS is replaying log"; return false; @@ -2656,56 +2666,68 @@ bool MDSRankDispatcher::kill_session(int64_t session_id, bool wait, std::strings std::string tmp = ss.str(); std::vector cmd = {tmp}; - C_SaferCond on_blacklist_inline; - Context *on_blacklist_done = nullptr; - if (wait) { - on_blacklist_done = &on_blacklist_inline; - } else { - on_blacklist_done = new FunctionContext([this, session_id](int r) { - objecter->wait_for_latest_osdmap( - new FunctionContext([this, session_id](int r) { - auto epoch = objecter->with_osdmap([](const OSDMap &o){ - return o.get_epoch(); - }); - - set_osd_epoch_barrier(epoch); - - Session *session = sessionmap.get_session( - entity_name_t(CEPH_ENTITY_TYPE_CLIENT, session_id)); - if (session) { - server->kill_session(session, NULL); - } else { - dout(1) << "session " << session_id << " was removed while we waited " - "for blacklist" << dendl; - } - })); - }); - } + auto kill_mds_session = [this, session_id, on_killed](){ + assert(mds_lock.is_locked_by_me()); + Session *session = sessionmap.get_session( + entity_name_t(CEPH_ENTITY_TYPE_CLIENT, session_id)); + if (session) { + if (on_killed) { + server->kill_session(session, on_killed); + } else { + C_SaferCond on_safe; + server->kill_session(session, &on_safe); - dout(4) << "Sending mon blacklist command: " << cmd[0] << dendl; - monc->start_mon_command(cmd, {}, nullptr, nullptr, on_blacklist_done); + mds_lock.Unlock(); + on_safe.wait(); + mds_lock.Lock(); + } + } else { + dout(1) << "session " << session_id << " was removed while we waited " + "for blacklist" << dendl; - if (wait) { - mds_lock.Unlock(); - int r = on_blacklist_inline.wait(); - mds_lock.Lock(); - if (r != 0) { - err_ss << "Failed to blacklist, r=" << r; - return false; + // Even though it wasn't us that removed it, kick our completion + // as the session has been removed. + if (on_killed) { + on_killed->complete(0); + } } + }; + + auto background_blacklist = [this, session_id, cmd](std::function fn){ + assert(mds_lock.is_locked_by_me()); + + Context *on_blacklist_done = new FunctionContext([this, session_id, fn](int r) { + objecter->wait_for_latest_osdmap( + new C_OnFinisher( + new FunctionContext([this, session_id, fn](int r) { + Mutex::Locker l(mds_lock); + auto epoch = objecter->with_osdmap([](const OSDMap &o){ + return o.get_epoch(); + }); + + set_osd_epoch_barrier(epoch); + + fn(); + }), finisher) + ); + }); + + dout(4) << "Sending mon blacklist command: " << cmd[0] << dendl; + monc->start_mon_command(cmd, {}, nullptr, nullptr, on_blacklist_done); + }; - // Wait for latest OSDMap so that we can learn the OSDMap epoch in - // which the blacklist happened - C_SaferCond on_latest_osdmap; - objecter->wait_for_latest_osdmap(&on_latest_osdmap); + auto blocking_blacklist = [this, cmd, &err_ss, background_blacklist](){ + C_SaferCond inline_ctx; + background_blacklist([&inline_ctx](){inline_ctx.complete(0);}); mds_lock.Unlock(); - on_latest_osdmap.wait(); + inline_ctx.wait(); mds_lock.Lock(); - auto epoch = objecter->with_osdmap([](const OSDMap &o){ - return o.get_epoch(); - }); + }; - set_osd_epoch_barrier(epoch); + if (wait) { + if (blacklist) { + blocking_blacklist(); + } // We dropped mds_lock, so check that session still exists session = sessionmap.get_session(entity_name_t(CEPH_ENTITY_TYPE_CLIENT, @@ -2715,13 +2737,13 @@ bool MDSRankDispatcher::kill_session(int64_t session_id, bool wait, std::strings "for blacklist" << dendl; return true; } - - C_SaferCond on_safe; - server->kill_session(session, &on_safe); - - mds_lock.Unlock(); - on_safe.wait(); - mds_lock.Lock(); + kill_mds_session(); + } else { + if (blacklist) { + background_blacklist(kill_mds_session); + } else { + kill_mds_session(); + } } return true; diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index af46f4c2f3e34..5fae1b43e9fee 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -406,6 +406,9 @@ class MDSRank { return map_targets.count(rank); } + bool kill_session(int64_t session_id, bool wait, bool blacklist, + std::stringstream& ss, Context *on_killed=nullptr); + protected: void dump_clientreplay_status(Formatter *f) const; void command_scrub_path(Formatter *f, const string& path, vector& scrubop_vec); @@ -535,7 +538,6 @@ public: Formatter *f, std::ostream& ss); void handle_mds_map(MMDSMap *m, MDSMap *oldmap); void handle_osd_map(); - bool kill_session(int64_t session_id, bool wait, std::stringstream& ss); void update_log_config(); bool handle_command(