client: detect blacklisting of myself

EIO all operations when this happens, and do a cut-down
unmount() so that once the user has noticed his mount
is broken, he can unmount it without getting stuck
waiting for flushes/caps/sessions.

Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 98c987b00e92ccecb24f78765074278b9269bef7..17ed10c8b88d8c0ff8c7c015a0608ad4eabcfc78 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -250,7 +250,7 @@ Client::Client(Messenger *m, MonClient *mc, Objecter *objecter_)
     whoami(mc->get_global_id()), cap_epoch_barrier(0),
     last_tid(0), oldest_tid(0), last_flush_tid(1),
     initialized(false),
-    mounted(false), unmounting(false),
+    mounted(false), unmounting(false), blacklisted(false),
     local_osd(-1), local_osd_epoch(0),
     unsafe_sync_write(0),
     client_lock("Client::client_lock")
@@ -293,6 +293,7 @@ Client::Client(Messenger *m, MonClient *mc, Objecter *objecter_)
                                  true));
   objecter_finisher.start();
   filer.reset(new Filer(objecter, &objecter_finisher));
+  objecter->enable_blacklist_events();
 }
 
 
@@ -1649,6 +1650,11 @@ int Client::make_request(MetaRequest *request,
     if (request->aborted())
       break;
 
+    if (blacklisted) {
+      request->abort(-EBLACKLISTED);
+      break;
+    }
+
     // set up wait cond
     Cond caller_cond;
     request->caller_cond = &caller_cond;
@@ -2398,6 +2404,46 @@ void Client::_handle_full_flag(int64_t pool)
 
 void Client::handle_osd_map(MOSDMap *m)
 {
+  std::set<entity_addr_t> new_blacklists;
+  objecter->consume_blacklist_events(&new_blacklists);
+
+  const auto myaddr = messenger->get_myaddr();
+  if (!blacklisted && new_blacklists.count(myaddr)) {
+    auto epoch = objecter->with_osdmap([](const OSDMap &o){
+        return o.get_epoch();
+        });
+    lderr(cct) << "I was blacklisted at osd epoch " << epoch << dendl;
+    blacklisted = true;
+    for (std::map<ceph_tid_t, MetaRequest*>::iterator p = mds_requests.begin();
+         p != mds_requests.end(); ) {
+      auto req = p->second;
+      ++p;
+      req->abort(-EBLACKLISTED);
+      if (req->caller_cond) {
+       req->kick = true;
+       req->caller_cond->Signal();
+      }
+    }
+
+    // Progress aborts on any requests that were on this waitlist.  Any
+    // requests that were on a waiting_for_open session waitlist
+    // will get kicked during close session below.
+    signal_cond_list(waiting_for_mdsmap);
+
+    // Force-close all sessions: assume this is not abandoning any state
+    // on the MDS side because the MDS will have seen the blacklist too.
+    while(!mds_sessions.empty()) {
+      auto i = mds_sessions.begin();
+      auto session = i->second;
+      _closed_mds_session(session);
+    }
+
+  } else if (blacklisted) {
+    // Handle case where we were blacklisted but no longer are
+    blacklisted = objecter->with_osdmap([myaddr](const OSDMap &o){
+        return o.is_blacklisted(myaddr);});
+  }
+
   if (objecter->osdmap_full_flag()) {
     _handle_full_flag(-1);
   } else {
@@ -5764,6 +5810,12 @@ void Client::unmount()
 
   _ll_drop_pins();
 
+  if (blacklisted) {
+    ldout(cct, 0) << " skipping clean shutdown, we are blacklisted" << dendl;
+    mounted = false;
+    return;
+  }
+
   while (unsafe_sync_write > 0) {
     ldout(cct, 0) << unsafe_sync_write << " unsafe_sync_writes, waiting"  << dendl;
     mount_cond.Wait(client_lock);

diff --git a/src/client/Client.h b/src/client/Client.h
index 7d2b73eb35d55d258ace5249fdd2fc2c76568cd5..da06729e8176dd21a35092be4ff37574398984ed 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -388,6 +388,7 @@ protected:
   bool   initialized;
   bool   mounted;
   bool   unmounting;
+  bool   blacklisted;
 
   // When an MDS has sent us a REJECT, remember that and don't
   // contact it again.  Remember which inst rejected us, so that