client: track flush TIDs for all pending flushing caps

Signed-off-by: Yan, Zheng <zyan@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 4259c6ecdf605cd244f776c26fc1a67f18a7739e..73e4ebdb7ddebc691716b01571e22e4f5fe82c77 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -2924,10 +2924,7 @@ void Client::send_cap(Inode *in, MetaSession *session, Cap *cap,
 
   if (flush) {
     flush_tid = ++in->last_flush_tid;
-    for (int i = 0; i < CEPH_CAP_BITS; ++i) {
-      if (flush & (1<<i))
-       in->flushing_cap_tid[i] = flush_tid;
-    }
+    in->flushing_cap_tids[flush_tid] = flush;
     follows = in->snaprealm->get_snap_context().seq;
   }
   
@@ -3585,8 +3582,10 @@ void Client::remove_session_caps(MetaSession *s)
     signal_cond_list(in->waitfor_caps);
     if (dirty_caps) {
       lderr(cct) << "remove_session_caps still has dirty|flushing caps on " << *in << dendl;
-      if (in->flushing_caps)
+      if (in->flushing_caps) {
        num_flushing_caps--;
+       in->flushing_cap_tids.clear();
+      }
       in->flushing_caps = 0;
       in->dirty_caps = 0;
       put_inode(in);
@@ -3763,20 +3762,17 @@ void Client::flush_caps(Inode *in, MetaSession *session)
           (cap->issued | cap->implemented), in->flushing_caps);
 }
 
-void Client::wait_sync_caps(Inode *in, uint16_t flush_tid[])
+void Client::wait_sync_caps(Inode *in, ceph_tid_t want)
 {
-retry:
-  for (int i = 0; i < CEPH_CAP_BITS; ++i) {
-    if (!(in->flushing_caps & (1 << i)))
-      continue;
-    // handle uint16_t wrapping
-    if ((int16_t)(in->flushing_cap_tid[i] - flush_tid[i]) <= 0) {
-      ldout(cct, 10) << "wait_sync_caps on " << *in << " flushing "
-                    << ccap_string(1 << i) << " want " << flush_tid[i]
-                    << " last " << in->flushing_cap_tid[i] << dendl;
-      wait_on_list(in->waitfor_caps);
-      goto retry;
-    }
+  while (in->flushing_caps) {
+    map<ceph_tid_t, int>::iterator it = in->flushing_cap_tids.begin();
+    assert(it != in->flushing_cap_tids.end());
+    if (it->first > want)
+      break;
+    ldout(cct, 10) << "wait_sync_caps on " << *in << " flushing "
+                  << ccap_string(it->second) << " want " << want
+                  << " last " << it->first << dendl;
+    wait_on_list(in->waitfor_caps);
   }
 }
 
@@ -4310,20 +4306,32 @@ void Client::handle_cap_trunc(MetaSession *session, Inode *in, MClientCaps *m)
 
 void Client::handle_cap_flush_ack(MetaSession *session, Inode *in, Cap *cap, MClientCaps *m)
 {
-  mds_rank_t mds = session->mds_num;
+  ceph_tid_t flush_ack_tid = m->get_client_tid();
   int dirty = m->get_dirty();
   int cleaned = 0;
-  uint16_t flush_ack_tid = static_cast<uint16_t>(m->get_client_tid());
-  for (int i = 0; i < CEPH_CAP_BITS; ++i) {
-    if ((dirty & (1 << i)) &&
-       (flush_ack_tid == in->flushing_cap_tid[i]))
-      cleaned |= 1 << i;
+  int flushed = 0;
+
+  for (map<ceph_tid_t, int>::iterator it = in->flushing_cap_tids.begin();
+       it != in->flushing_cap_tids.end(); ) {
+    if (it->first == flush_ack_tid)
+      cleaned = it->second;
+    if (it->first <= flush_ack_tid) {
+      in->flushing_cap_tids.erase(it++);
+      ++flushed;
+      continue;
+    }
+    cleaned &= ~it->second;
+    if (!cleaned)
+      break;
+    ++it;
   }
 
-  ldout(cct, 5) << "handle_cap_flush_ack mds." << mds
+  ldout(cct, 5) << "handle_cap_flush_ack mds." << session->mds_num
          << " cleaned " << ccap_string(cleaned) << " on " << *in
          << " with " << ccap_string(dirty) << dendl;
 
+  if (flushed)
+    signal_cond_list(in->waitfor_caps);
 
   if (!cleaned) {
     ldout(cct, 10) << " tid " << m->get_client_tid() << " != any cap bit tids" << dendl;
@@ -4338,7 +4346,6 @@ void Client::handle_cap_flush_ack(MetaSession *session, Inode *in, Cap *cap, MCl
        num_flushing_caps--;
        sync_cond.Signal();
       }
-      signal_cond_list(in->waitfor_caps);
       if (!in->caps_dirty())
        put_inode(in);
     }
@@ -7804,7 +7811,6 @@ int Client::fsync(int fd, bool syncdataonly)
 int Client::_fsync(Inode *in, bool syncdataonly)
 {
   int r = 0;
-  uint16_t wait_on_flush[CEPH_CAP_BITS];
   bool flushed_metadata = false;
   Mutex lock("Client::_fsync::lock");
   Cond cond;
@@ -7823,10 +7829,8 @@ int Client::_fsync(Inode *in, bool syncdataonly)
   
   if (!syncdataonly && (in->dirty_caps & ~CEPH_CAP_ANY_FILE_WR)) {
     check_caps(in, true);
-    if (in->flushing_caps) {
+    if (in->flushing_caps)
       flushed_metadata = true;
-      memcpy(wait_on_flush, in->flushing_cap_tid, sizeof(wait_on_flush));
-    }
   } else ldout(cct, 10) << "no metadata needs to commit" << dendl;
 
   if (object_cacher_completion) { // wait on a real reply instead of guessing
@@ -7864,7 +7868,7 @@ int Client::_fsync(Inode *in, bool syncdataonly)
 
   if (!r) {
     if (flushed_metadata)
-      wait_sync_caps(in, wait_on_flush);
+      wait_sync_caps(in, in->last_flush_tid);
 
     ldout(cct, 10) << "ino " << in->ino << " has no uncommitted writes" << dendl;
   } else {

diff --git a/src/client/Client.h b/src/client/Client.h
index 78a2da7ee04a86fb13188e307a5eeff1ab685274..f560e662b3363a11c5c003115958dcfaa138d8b7 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -561,7 +561,7 @@ protected:
   void get_cap_ref(Inode *in, int cap);
   void put_cap_ref(Inode *in, int cap);
   void flush_snaps(Inode *in, bool all_again=false, CapSnap *again=0);
-  void wait_sync_caps(Inode *in, uint16_t flush_tid[]);
+  void wait_sync_caps(Inode *in, ceph_tid_t want);
   void wait_sync_caps(uint64_t want);
   void queue_cap_snap(Inode *in, SnapContext &old_snapc);
   void finish_cap_snap(Inode *in, CapSnap *capsnap, int used);

diff --git a/src/client/Inode.cc b/src/client/Inode.cc
index 16eee7a7aa494b41fac535a469ae3dc49fce923d..994c2971ca173c44f6f0a29fd7ed539720ae1d2f 100644 (file)
--- a/src/client/Inode.cc
+++ b/src/client/Inode.cc
@@ -396,11 +396,11 @@ void Inode::dump(Formatter *f) const
     f->dump_stream("flushings_caps") << ccap_string(flushing_caps);
     f->dump_unsigned("flushing_cap_seq", flushing_cap_seq);
     f->open_object_section("flushing_cap_tid");
-    for (unsigned bit = 0; bit < CEPH_CAP_BITS; bit++) {
-      if (flushing_caps & (1 << bit)) {
-       string n(ccap_string(1 << bit));
-       f->dump_unsigned(n.c_str(), flushing_cap_tid[bit]);
-      }
+    for (map<ceph_tid_t, int>::const_iterator p = flushing_cap_tids.begin();
+        p != flushing_cap_tids.end();
+        ++p) {
+      string n(ccap_string(p->second));
+      f->dump_unsigned(n.c_str(), p->first);
     }
     f->close_section();
   }

diff --git a/src/client/Inode.h b/src/client/Inode.h
index f18f65272d93176b1c461d3244e3235a25938edf..975a48b00638e98fb07225748902581248f395e4 100644 (file)
--- a/src/client/Inode.h
+++ b/src/client/Inode.h
@@ -234,7 +234,7 @@ struct Inode {
   Cap *auth_cap;
   unsigned dirty_caps, flushing_caps;
   uint64_t flushing_cap_seq;
-  __u16 flushing_cap_tid[CEPH_CAP_BITS];
+  std::map<ceph_tid_t, int> flushing_cap_tids;
   int shared_gen, cache_gen;
   int snap_caps, snap_cap_refs;
   utime_t hold_caps_until;
@@ -316,7 +316,6 @@ struct Inode {
   {
     memset(&dir_layout, 0, sizeof(dir_layout));
     memset(&layout, 0, sizeof(layout));
-    memset(&flushing_cap_tid, 0, sizeof(__u16)*CEPH_CAP_BITS);
     memset(&quota, 0, sizeof(quota));
   }
   ~Inode() { }