mds: wait acknowledgment for export abort notification

To disambiguate other mds's failed import, survivor bystander mds
need to receive either exporter mds' export abort notification or
exporter mds' resolve message. For bystander mds, it's hard to
distinguish "export succeeded" from the case "hasn't received
export abort notification".

To handle this problem, we rely on the fact that surviver mds does
not send resolve message to the recovering mds until it finishes
all its exports. Without the resolve message, the recovering mds
can't go to rejoin state. So when mds cluster is in rejoin state,
we know all mds have finished their exports. If export abort
notifications also require acknowledgments. When mds cluster is
in rejoin state, we know all export abort notifications have been
proceesed by bystander mds. So bystander mds can disambiguate other
mds' imports

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index 8af8344c21a409c222afbe54c8e27fd7b8a32562..1fe1c862848217a2790164d9c12817bef7be2b5b 100644 (file)
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -295,13 +295,15 @@ void Migrator::export_try_cancel(CDir *dir, bool notify_peer)
     // NOTE: state order reversal, warning comes after prepping
   case EXPORT_WARNING:
     dout(10) << "export state=warning : unpinning bounds, unfreezing, notifying" << dendl;
+    it->second.state = EXPORT_CANCELLING;
     // fall-thru
 
   case EXPORT_PREPPING:
-    if (state != EXPORT_WARNING)
+    if (state != EXPORT_WARNING) {
       dout(10) << "export state=prepping : unpinning bounds, unfreezing" << dendl;
+      it->second.state = EXPORT_CANCELLED;
+    }
 
-    it->second.state = EXPORT_CANCELLED;
     {
       // unpin bounds
       set<CDir*> bounds;
@@ -329,7 +331,7 @@ void Migrator::export_try_cancel(CDir *dir, bool notify_peer)
 
   case EXPORT_EXPORTING:
     dout(10) << "export state=exporting : reversing, and unfreezing" << dendl;
-    it->second.state = EXPORT_CANCELLED;
+    it->second.state = EXPORT_CANCELLING;
     export_reverse(dir);
     break;
 
@@ -338,20 +340,25 @@ void Migrator::export_try_cancel(CDir *dir, bool notify_peer)
     dout(10) << "export state=loggingfinish|notifying : ignoring dest failure, we were successful." << dendl;
     // leave export_state, don't clean up now.
     break;
+  case EXPORT_CANCELLING:
+    break;
 
   default:
     ceph_abort();
   }
 
   // finish clean-up?
-  if (it->second.state == EXPORT_CANCELLED) {
-    MutationRef mut = it->second.mut;
-
-    export_state.erase(it);
-    dir->state_clear(CDir::STATE_EXPORTING);
-
-    // send pending import_maps?  (these need to go out when all exports have finished.)
-    cache->maybe_send_pending_resolves();
+  if (it->second.state == EXPORT_CANCELLING ||
+      it->second.state == EXPORT_CANCELLED) {
+    MutationRef mut;
+    mut.swap(it->second.mut);
+
+    if (it->second.state == EXPORT_CANCELLED) {
+      export_state.erase(it);
+      dir->state_clear(CDir::STATE_EXPORTING);
+      // send pending import_maps?
+      cache->maybe_send_pending_resolves();
+    }
 
     // drop locks
     if (state == EXPORT_LOCKING || state == EXPORT_DISCOVERING) {
@@ -370,6 +377,17 @@ void Migrator::export_try_cancel(CDir *dir, bool notify_peer)
   }
 }
 
+void Migrator::export_cancel_finish(CDir *dir)
+{
+  assert(dir->state_test(CDir::STATE_EXPORTING));
+  dir->state_clear(CDir::STATE_EXPORTING);
+
+  // pinned by Migrator::export_notify_abort()
+  dir->auth_unpin(this);
+  // send pending import_maps?  (these need to go out when all exports have finished.)
+  cache->maybe_send_pending_resolves();
+}
+
 // ==========================================================
 // mds failure handling
 
@@ -406,7 +424,8 @@ void Migrator::handle_mds_failure_or_stop(mds_rank_t who)
     //  - that are going to the failed node
     //  - that aren't frozen yet (to avoid auth_pin deadlock)
     //  - they havne't prepped yet (they may need to discover bounds to do that)
-    if (p->second.peer == who ||
+    if ((p->second.peer == who &&
+        p->second.state != EXPORT_CANCELLING) ||
        p->second.state == EXPORT_LOCKING ||
        p->second.state == EXPORT_DISCOVERING ||
        p->second.state == EXPORT_FREEZING ||
@@ -415,11 +434,11 @@ void Migrator::handle_mds_failure_or_stop(mds_rank_t who)
       dout(10) << "cleaning up export state (" << p->second.state << ")"
               << get_export_statename(p->second.state) << " of " << *dir << dendl;
       export_try_cancel(dir);
-    } else {
+    } else if (p->second.peer != who) {
       // bystander failed.
       if (p->second.warning_ack_waiting.erase(who)) {
-       p->second.notify_ack_waiting.erase(who);   // they won't get a notify either.
        if (p->second.state == EXPORT_WARNING) {
+         p->second.notify_ack_waiting.erase(who);   // they won't get a notify either.
          // exporter waiting for warning acks, let's fake theirs.
          dout(10) << "faking export_warning_ack from mds." << who
                   << " on " << *dir << " to mds." << p->second.peer
@@ -429,13 +448,18 @@ void Migrator::handle_mds_failure_or_stop(mds_rank_t who)
        }
       }
       if (p->second.notify_ack_waiting.erase(who)) {
+       // exporter is waiting for notify acks, fake it
+       dout(10) << "faking export_notify_ack from mds." << who
+                << " on " << *dir << " to mds." << p->second.peer
+                << dendl;
        if (p->second.state == EXPORT_NOTIFYING) {
-         // exporter is waiting for notify acks, fake it
-         dout(10) << "faking export_notify_ack from mds." << who
-                  << " on " << *dir << " to mds." << p->second.peer
-                  << dendl;
          if (p->second.notify_ack_waiting.empty())
            export_finish(dir);
+       } else if (p->second.state == EXPORT_CANCELLING) {
+         if (p->second.notify_ack_waiting.empty()) {
+           export_state.erase(p);
+           export_cancel_finish(dir);
+         }
        }
       }
     }
@@ -628,7 +652,9 @@ void Migrator::audit()
     CDir *dir = p->first;
     if (p->second.state == EXPORT_LOCKING ||
        p->second.state == EXPORT_DISCOVERING ||
-       p->second.state == EXPORT_FREEZING) continue;
+       p->second.state == EXPORT_FREEZING ||
+       p->second.state == EXPORT_CANCELLING)
+      continue;
     assert(dir->is_ambiguous_dir_auth());
     assert(dir->authority().first  == mds->get_nodeid() ||
           dir->authority().second == mds->get_nodeid());
@@ -899,7 +925,9 @@ void Migrator::export_sessions_flushed(CDir *dir, uint64_t tid)
   dout(7) << "export_sessions_flushed " << *dir << dendl;
 
   map<CDir*,export_state_t>::iterator it = export_state.find(dir);
-  if (it == export_state.end() || it->second.tid != tid) {
+  if (it == export_state.end() ||
+      it->second.state == EXPORT_CANCELLING ||
+      it->second.tid != tid) {
     // export must have aborted.
     dout(7) << "export must have aborted on " << dir << dendl;
     return;
@@ -1206,6 +1234,7 @@ void Migrator::export_go_synced(CDir *dir, uint64_t tid)
 
   map<CDir*,export_state_t>::iterator it = export_state.find(dir);
   if (it == export_state.end() ||
+      it->second.state == EXPORT_CANCELLING ||
       it->second.tid != tid) {
     // export must have aborted.  
     dout(7) << "export must have aborted on " << dir << dendl;
@@ -1606,11 +1635,19 @@ void Migrator::export_notify_abort(CDir *dir, set<CDir*>& bounds)
   dout(7) << "export_notify_abort " << *dir << dendl;
 
   export_state_t& stat = export_state[dir];
+  assert(stat.state == EXPORT_CANCELLING);
+
+  if (stat.notify_ack_waiting.empty()) {
+    stat.state = EXPORT_CANCELLED;
+    return;
+  }
+
+  dir->auth_pin(this);
 
   for (set<mds_rank_t>::iterator p = stat.notify_ack_waiting.begin();
        p != stat.notify_ack_waiting.end();
        ++p) {
-    MExportDirNotify *notify = new MExportDirNotify(dir->dirfrag(),stat.tid,false,
+    MExportDirNotify *notify = new MExportDirNotify(dir->dirfrag(),stat.tid, true,
                                                    pair<int,int>(mds->get_nodeid(),stat.peer),
                                                    pair<int,int>(mds->get_nodeid(),CDIR_AUTH_UNKNOWN));
     for (set<CDir*>::iterator i = bounds.begin(); i != bounds.end(); ++i)
@@ -1761,22 +1798,29 @@ void Migrator::handle_export_notify_ack(MExportDirNotifyAck *m)
   auto export_state_entry = export_state.find(dir);
   if (export_state_entry != export_state.end()) {
     export_state_t& stat = export_state_entry->second;
-    if (stat.state == EXPORT_WARNING) {
+    if (stat.state == EXPORT_WARNING &&
+       stat.warning_ack_waiting.erase(from)) {
       // exporting. process warning.
       dout(7) << "handle_export_notify_ack from " << m->get_source()
              << ": exporting, processing warning on " << *dir << dendl;
-      stat.warning_ack_waiting.erase(from);
-
       if (stat.warning_ack_waiting.empty())
        export_go(dir);     // start export.
-    } else if (stat.state == EXPORT_NOTIFYING) {
+    } else if (stat.state == EXPORT_NOTIFYING &&
+              stat.notify_ack_waiting.erase(from)) {
       // exporting. process notify.
       dout(7) << "handle_export_notify_ack from " << m->get_source()
              << ": exporting, processing notify on " << *dir << dendl;
-      stat.notify_ack_waiting.erase(from);
-
       if (stat.notify_ack_waiting.empty())
        export_finish(dir);
+    } else if (stat.state == EXPORT_CANCELLING &&
+              m->get_new_auth().second == CDIR_AUTH_UNKNOWN && // not warning ack
+              stat.notify_ack_waiting.erase(from)) {
+      dout(7) << "handle_export_notify_ack from " << m->get_source()
+             << ": cancelling export, processing notify on " << *dir << dendl;
+      if (stat.notify_ack_waiting.empty()) {
+       export_state.erase(export_state_entry);
+       export_cancel_finish(dir);
+      }
     }
   }
   else {
@@ -3017,7 +3061,7 @@ void Migrator::handle_export_notify(MExportDirNotify *m)
   
   // send ack
   if (m->wants_ack()) {
-    mds->send_message_mds(new MExportDirNotifyAck(m->get_dirfrag(), m->get_tid()), from);
+    mds->send_message_mds(new MExportDirNotifyAck(m->get_dirfrag(), m->get_tid(), m->get_new_auth()), from);
   } else {
     // aborted.  no ack.
     dout(7) << "handle_export_notify no ack requested" << dendl;

diff --git a/src/mds/Migrator.h b/src/mds/Migrator.h
index 3c6cc602b87d52db3531d2494574b4d2ac371ae5..85ae9cf44104f4e75adadf18e15930ddfd28cfb9 100644 (file)
--- a/src/mds/Migrator.h
+++ b/src/mds/Migrator.h
@@ -58,16 +58,18 @@ private:
 public:
   // export stages.  used to clean up intelligently if there's a failure.
   const static int EXPORT_CANCELLED    = 0;  // cancelled
-  const static int EXPORT_LOCKING      = 1;  // acquiring locks
-  const static int EXPORT_DISCOVERING  = 2;  // dest is disovering export dir
-  const static int EXPORT_FREEZING     = 3;  // we're freezing the dir tree
-  const static int EXPORT_PREPPING     = 4;  // sending dest spanning tree to export bounds
-  const static int EXPORT_WARNING      = 5;  // warning bystanders of dir_auth_pending
-  const static int EXPORT_EXPORTING    = 6;  // sent actual export, waiting for ack
-  const static int EXPORT_LOGGINGFINISH        = 7;  // logging EExportFinish
-  const static int EXPORT_NOTIFYING    = 8;  // waiting for notifyacks
+  const static int EXPORT_CANCELLING   = 1;  // waiting for cancel notifyacks
+  const static int EXPORT_LOCKING      = 2;  // acquiring locks
+  const static int EXPORT_DISCOVERING  = 3;  // dest is disovering export dir
+  const static int EXPORT_FREEZING     = 4;  // we're freezing the dir tree
+  const static int EXPORT_PREPPING     = 5;  // sending dest spanning tree to export bounds
+  const static int EXPORT_WARNING      = 6;  // warning bystanders of dir_auth_pending
+  const static int EXPORT_EXPORTING    = 7;  // sent actual export, waiting for ack
+  const static int EXPORT_LOGGINGFINISH        = 8;  // logging EExportFinish
+  const static int EXPORT_NOTIFYING    = 9;  // waiting for notifyacks
   static const char *get_export_statename(int s) {
     switch (s) {
+    case EXPORT_CANCELLING: return "cancelling";
     case EXPORT_LOCKING: return "locking";
     case EXPORT_DISCOVERING: return "discovering";
     case EXPORT_FREEZING: return "freezing";
@@ -274,6 +276,7 @@ public:
   void export_go(CDir *dir);
   void export_go_synced(CDir *dir, uint64_t tid);
   void export_try_cancel(CDir *dir, bool notify_peer=true);
+  void export_cancel_finish(CDir *dir);
   void export_reverse(CDir *dir);
   void export_notify_abort(CDir *dir, set<CDir*>& bounds);
   void handle_export_ack(MExportDirAck *m);

diff --git a/src/messages/MExportDirNotifyAck.h b/src/messages/MExportDirNotifyAck.h
index aa246597b0422ac75802f6c39459e42ea75605cf..0695d3685e30b506dca2dcb319062e4755d32ccc 100644 (file)
--- a/src/messages/MExportDirNotifyAck.h
+++ b/src/messages/MExportDirNotifyAck.h
@@ -19,13 +19,15 @@
 
 class MExportDirNotifyAck : public Message {
   dirfrag_t dirfrag;
+  pair<__s32,__s32> new_auth;
 
  public:
   dirfrag_t get_dirfrag() { return dirfrag; }
+  pair<__s32,__s32> get_new_auth() { return new_auth; }
   
   MExportDirNotifyAck() {}
-  MExportDirNotifyAck(dirfrag_t df, uint64_t tid) :
-    Message(MSG_MDS_EXPORTDIRNOTIFYACK), dirfrag(df) {
+  MExportDirNotifyAck(dirfrag_t df, uint64_t tid, pair<__s32,__s32> na) :
+    Message(MSG_MDS_EXPORTDIRNOTIFYACK), dirfrag(df), new_auth(na) {
     set_tid(tid);
   }
 private:
@@ -39,10 +41,12 @@ public:
 
   void encode_payload(uint64_t features) {
     ::encode(dirfrag, payload);
+    ::encode(new_auth, payload);
   }
   void decode_payload() {
     bufferlist::iterator p = payload.begin();
     ::decode(dirfrag, p);
+    ::decode(new_auth, p);
   }
   
 };