client: add imported caps when handling cap export message

Version 3 cap export message includes information about the imported
caps. It allows us to add the imported caps if the corresponding cap
import message still hasn't been received.

This allow us to handle situation that the importer MDS crashes and
the cap import message is missing.

Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 9ff6d016d85c5ae7821a314fe65f9f57a9331133..b243ca1e7164fc48f064ba8669de7d3509854337 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -2012,18 +2012,6 @@ void Client::send_reconnect(MetaSession *session)
        did_snaprealm.insert(in->snaprealm->ino);
       }        
     }
-    if (in->exporting_mds == mds) {
-      ldout(cct, 10) << " clearing exporting_caps on " << p->first << dendl;
-      in->exporting_mds = -1;
-      in->exporting_issued = 0;
-      in->exporting_mseq = 0;
-      if (!in->is_any_caps()) {
-       ldout(cct, 10) << "  removing last cap, closing snaprealm" << dendl;
-       in->snaprealm_item.remove_myself();
-       put_snap_realm(in->snaprealm);
-       in->snaprealm = 0;
-      }
-    }
   }
   
   // reset my cap seq number
@@ -2898,6 +2886,24 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id
   int mds = mds_session->mds_num;
   if (in->caps.count(mds)) {
     cap = in->caps[mds];
+
+    /*
+     * auth mds of the inode changed. we received the cap export
+     * message, but still haven't received the cap import message.
+     * handle_cap_export() updated the new auth MDS' cap.
+     *
+     * "ceph_seq_cmp(seq, cap->seq) <= 0" means we are processing
+     * a message that was send before the cap import message. So
+     * don't remove caps.
+     */
+    if (ceph_seq_cmp(seq, cap->seq) <= 0) {
+      assert(cap == in->auth_cap);
+      assert(cap->cap_id == cap_id);
+      seq = cap->seq;
+      mseq = cap->mseq;
+      issued |= cap->issued;
+      flags |= CEPH_CAP_FLAG_AUTH;
+    }
   } else {
     mds_session->num_caps++;
     if (!in->is_any_caps()) {
@@ -2906,12 +2912,6 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id
       in->snaprealm->inodes_with_caps.push_back(&in->snaprealm_item);
       ldout(cct, 15) << "add_update_cap first one, opened snaprealm " << in->snaprealm << dendl;
     }
-    if (in->exporting_mds == mds) {
-      ldout(cct, 10) << "add_update_cap clearing exporting_caps on " << mds << dendl;
-      in->exporting_mds = -1;
-      in->exporting_issued = 0;
-      in->exporting_mseq = 0;
-    }
     in->caps[mds] = cap = new Cap;
     mds_session->caps.push_back(&cap->cap_item);
     cap->session = mds_session;
@@ -3505,12 +3505,10 @@ void Client::handle_cap_import(MetaSession *session, Inode *in, MClientCaps *m)
                 m->get_caps(), m->get_seq(), m->get_mseq(), m->get_realm(),
                 CEPH_CAP_FLAG_AUTH);
 
-  if (m->peer.cap_id) {
+  if (m->peer.cap_id && in->caps.count(m->peer.mds)) {
     Cap *cap = in->caps[m->peer.mds];
-    if (cap && cap->cap_id == m->peer.cap_id) {
-      in->exporting_issued = cap->issued;
+    if (cap && cap->cap_id == m->peer.cap_id)
       remove_cap(cap, (m->peer.flags & CEPH_CAP_FLAG_RELEASE));
-    }
   }
   
   if (in->auth_cap && in->auth_cap->session->mds_num == mds) {
@@ -3525,44 +3523,39 @@ void Client::handle_cap_import(MetaSession *session, Inode *in, MClientCaps *m)
 void Client::handle_cap_export(MetaSession *session, Inode *in, MClientCaps *m)
 {
   int mds = session->mds_num;
-  Cap *cap = NULL;
 
-  // note?
-  bool found_higher_mseq = false;
-  for (map<int,Cap*>::iterator p = in->caps.begin();
-       p != in->caps.end();
-       ++p) {
-    if (p->first == mds)
-      cap = p->second;
-    if (ceph_seq_cmp(p->second->mseq, m->get_mseq()) > 0) {
-      found_higher_mseq = true;
-      ldout(cct, 5) << "handle_cap_export ino " << m->get_ino() << " mseq " << m->get_mseq() 
-             << " EXPORT from mds." << mds
-             << ", but mds." << p->first << " has higher mseq " << p->second->mseq << dendl;
-    }
-  }
-
-  if (cap) {
-    if (!found_higher_mseq) {
-      ldout(cct, 5) << "handle_cap_export ino " << m->get_ino() << " mseq " << m->get_mseq() 
-             << " EXPORT from mds." << mds
-             << ", setting exporting_issued " << ccap_string(cap->issued) << dendl;
-      in->exporting_issued = cap->issued;
-      in->exporting_mseq = m->get_mseq();
-      in->exporting_mds = mds;
-    } else 
-      ldout(cct, 5) << "handle_cap_export ino " << m->get_ino() << " mseq " << m->get_mseq() 
-             << " EXPORT from mds." << mds
-             << ", just removing old cap" << dendl;
+  ldout(cct, 5) << "handle_cap_export ino " << m->get_ino() << " mseq " << m->get_mseq()
+               << " EXPORT from mds." << mds << dendl;
+
+  if (in->caps.count(mds)) {
+    Cap *cap = in->caps[mds];
+
+    if (m->peer.cap_id) {
+      MetaSession *tsession = _get_or_open_mds_session(m->peer.mds);
+      if (in->caps.count(m->peer.mds)) {
+       Cap *tcap = in->caps[m->peer.mds];
+       if (tcap->cap_id != m->peer.cap_id ||
+           ceph_seq_cmp(tcap->seq, m->peer.seq) < 0) {
+         tcap->cap_id = m->peer.cap_id;
+         tcap->seq = m->peer.seq - 1;
+         tcap->issue_seq = tcap->seq;
+         tcap->mseq = m->peer.mseq;
+         tcap->issued |= cap->issued;
+         tcap->implemented |= cap->issued;
+         if (cap == in->auth_cap)
+           in->auth_cap = tcap;
+         if (in->auth_cap == tcap && in->flushing_cap_item.is_on_list())
+           tsession->flushing_caps.push_back(&in->flushing_cap_item);
+       }
+      } else {
+       add_update_cap(in, tsession, m->peer.cap_id, cap->issued,
+                      m->peer.seq - 1, m->peer.mseq, (uint64_t)-1,
+                      cap == in->auth_cap ? CEPH_CAP_FLAG_AUTH : 0);
+      }
+    }
 
     remove_cap(cap, false);
   }
-  // else we already released it
-
-  // open export targets, so we'll get the matching IMPORT, even if we
-  // have seen a newer import (or have released the cap entirely), as there
-  // may be an intervening revocation that will otherwise get blocked up.
-  connect_mds_targets(mds);
 
   m->put();
 }
@@ -3748,7 +3741,7 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient
   }
 
   bool check = false;
-  if (m->get_op() == CEPH_CAP_OP_IMPORT && m->get_wanted() != wanted) {
+  if (m->get_op() == CEPH_CAP_OP_IMPORT && m->get_wanted() != wanted)
     check = true;
 
   check_cap_issue(in, cap, issued);

diff --git a/src/client/Inode.cc b/src/client/Inode.cc
index 4e53e14bd24499c4cbb7811792430a0beac513be..0d9d9d9a45baebcbe8c5f63e5ed6ee8eefe1ce61 100644 (file)
--- a/src/client/Inode.cc
+++ b/src/client/Inode.cc
@@ -137,7 +137,7 @@ bool Inode::put_cap_ref(int cap)
 
 bool Inode::is_any_caps()
 {
-  return caps.size() || exporting_mds >= 0;
+  return caps.size();
 }
 
 bool Inode::cap_is_valid(Cap* cap)
@@ -155,7 +155,7 @@ bool Inode::cap_is_valid(Cap* cap)
 
 int Inode::caps_issued(int *implemented)
 {
-  int c = exporting_issued | snap_caps;
+  int c = snap_caps;
   int i = 0;
   for (map<int,Cap*>::iterator it = caps.begin();
        it != caps.end();
@@ -183,7 +183,7 @@ void Inode::try_touch_cap(int mds)
 
 bool Inode::caps_issued_mask(unsigned mask)
 {
-  int c = exporting_issued | snap_caps;
+  int c = snap_caps;
   if ((c & mask) == mask)
     return true;
   // prefer auth cap
@@ -390,10 +390,6 @@ void Inode::dump(Formatter *f) const
     f->dump_int("snap_caps", snap_caps);
     f->dump_int("snap_cap_refs", snap_cap_refs);
   }
-  if (exporting_issued || exporting_mseq) {
-    f->dump_stream("exporting_issued") << ccap_string(exporting_issued);
-    f->dump_int("exporting_mseq", exporting_mds);
-  }
 
   f->dump_stream("hold_caps_until") << hold_caps_until;
   f->dump_unsigned("last_flush_tid", last_flush_tid);

diff --git a/src/client/Inode.h b/src/client/Inode.h
index b6c7a38aa0be762a3425d452ebe7b41919f831df..692e76ab9023a5f8ebae7760c9b2a83775f1638e 100644 (file)
--- a/src/client/Inode.h
+++ b/src/client/Inode.h
@@ -140,9 +140,6 @@ class Inode {
   __u16 flushing_cap_tid[CEPH_CAP_BITS];
   int shared_gen, cache_gen;
   int snap_caps, snap_cap_refs;
-  unsigned exporting_issued;
-  int exporting_mds;
-  ceph_seq_t exporting_mseq;
   utime_t hold_caps_until;
   xlist<Inode*>::item cap_item, flushing_cap_item;
   tid_t last_flush_tid;
@@ -216,7 +213,6 @@ class Inode {
       dir_hashed(false), dir_replicated(false), auth_cap(NULL),
       dirty_caps(0), flushing_caps(0), flushing_cap_seq(0), shared_gen(0), cache_gen(0),
       snap_caps(0), snap_cap_refs(0),
-      exporting_issued(0), exporting_mds(-1), exporting_mseq(0),
       cap_item(this), flushing_cap_item(this), last_flush_tid(0),
       snaprealm(0), snaprealm_item(this), snapdir_parent(0),
       oset((void *)this, newlayout->fl_pg_pool, ino),