mds: apply OSDMap blacklist to CephFS clients

Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 7aa53dcc04651b3f6d999e1f8223cd1372d82ae3..e4a3b05553b30dc8a5a753e71d2c3a7bff09108f 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -1338,6 +1338,18 @@ void MDSRank::reconnect_start()
     reopen_log();
   }
 
+  // Drop any blacklisted clients from the SessionMap before going
+  // into reconnect, so that we don't wait for them.
+  objecter->enable_blacklist_events();
+  std::set<entity_addr_t> blacklist;
+  objecter->with_osdmap([this, &blacklist](const OSDMap& o) {
+      o.get_blacklist(&blacklist);
+  });
+  auto killed = server->apply_blacklist(blacklist);
+  dout(4) << "reconnect_start: killed " << killed << " blacklisted sessions ("
+          << blacklist.size() << " blacklist entries, "
+          << sessionmap.get_sessions().size() << ")" << dendl;
+
   server->reconnect_clients(new C_MDS_VoidFn(this, &MDSRank::reconnect_done));
   finish_contexts(g_ceph_context, waiting_for_reconnect);
 }
@@ -1508,6 +1520,9 @@ void MDSRank::boot_create()
   mdlog->journal_segment_subtree_map(fin.new_sub());
   mdlog->flush();
 
+  // Usually we do this during reconnect, but creation skips that.
+  objecter->enable_blacklist_events();
+
   fin.activate();
 }
 
@@ -2599,6 +2614,13 @@ void MDSRankDispatcher::handle_osd_map()
 
   purge_queue.update_op_limit(*mdsmap);
 
+  std::set<entity_addr_t> newly_blacklisted;
+  objecter->consume_blacklist_events(&newly_blacklisted);
+  auto epoch = objecter->with_osdmap([](const OSDMap &o){return o.get_epoch();});
+  dout(4) << "handle_osd_map epoch " << epoch << ", "
+          << newly_blacklisted.size() << " new blacklist entries" << dendl;
+  server->apply_blacklist(newly_blacklisted);
+
   // By default the objecter only requests OSDMap updates on use,
   // we would like to always receive the latest maps in order to
   // apply policy based on the FULL flag.

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 500fd3ec629c2a91d996e9d3bd142c58f1deae38..18a67801d6b0cfa14e452096f979a8c8cc870c04 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -318,6 +318,7 @@ Session *Server::get_session(Message *m)
 void Server::handle_client_session(MClientSession *m)
 {
   version_t pv;
+  bool blacklisted = false;
   Session *session = get_session(m);
 
   dout(3) << "handle_client_session " << *m << " from " << m->get_source() << dendl;
@@ -346,6 +347,17 @@ void Server::handle_client_session(MClientSession *m)
     assert(session->is_closed() ||
           session->is_closing());
 
+    blacklisted = mds->objecter->with_osdmap(
+        [session](const OSDMap &osd_map) -> bool {
+          return osd_map.is_blacklisted(session->info.inst.addr);
+        });
+
+    if (blacklisted) {
+      dout(10) << "ignoring blacklisted client " << session->info.inst.addr << dendl;
+      m->put();
+      return;
+    }
+
     session->set_client_metadata(m->client_meta);
     dout(20) << __func__ << " CEPH_SESSION_REQUEST_OPEN "
       << session->info.client_metadata.size() << " metadata entries:" << dendl;
@@ -761,6 +773,32 @@ void Server::kill_session(Session *session, Context *on_safe)
   }
 }
 
+size_t Server::apply_blacklist(const std::set<entity_addr_t> &blacklist)
+{
+  std::list<Session*> victims;
+  const auto sessions = mds->sessionmap.get_sessions();
+  for (const auto p : sessions)  {
+    if (!p.first.is_client()) {
+      // Do not apply OSDMap blacklist to MDS daemons, we find out
+      // about their death via MDSMap.
+      continue;
+    }
+
+    Session *s = p.second;
+    if (blacklist.count(s->info.inst.addr)) {
+      victims.push_back(s);
+    }
+  }
+
+  for (const auto s : victims) {
+    kill_session(s, nullptr);
+  }
+
+  dout(10) << "apply_blacklist: killed " << victims.size() << dendl;
+
+  return victims.size();
+}
+
 void Server::journal_close_session(Session *session, int state, Context *on_safe)
 {
   uint64_t sseq = mds->sessionmap.set_state(session, state);

diff --git a/src/mds/Server.h b/src/mds/Server.h
index 7d1aaeab7452dea61096897a93e2c86895f98da2..84947856ed21303f14b29bbf1b4195b4ed33d8c1 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -122,6 +122,7 @@ public:
   void terminate_sessions();
   void find_idle_sessions();
   void kill_session(Session *session, Context *on_safe);
+  size_t apply_blacklist(const std::set<entity_addr_t> &blacklist);
   void journal_close_session(Session *session, int state, Context *on_safe);
   void reconnect_clients(MDSInternalContext *reconnect_done_);
   void handle_client_reconnect(class MClientReconnect *m);