client: make wait_sync_caps(uint64_t) check per-session flush TIDs

Now we have data structure to track per-session flush TIDs. It's
easy to check if all cap flushes before a TID are completed

Signed-off-by: Yan, Zheng <zyan@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index d98e514f2ab2da5b99981b1532c6469d56dfa51e..d6014c84db487f98b09289ff59833d2ce0f5902c 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -179,7 +179,7 @@ Client::Client(Messenger *m, MonClient *mc)
     tick_event(NULL),
     monclient(mc), messenger(m), whoami(m->get_myname().num()),
     cap_epoch_barrier(0),
-    last_tid(0), oldest_tid(0), last_flush_seq(0), last_flush_tid(1),
+    last_tid(0), oldest_tid(0), last_flush_tid(1),
     initialized(false), authenticated(false),
     mounted(false), unmounting(false),
     local_osd(-1), local_osd_epoch(0),
@@ -3721,7 +3721,6 @@ int Client::mark_caps_flushing(Inode *in, ceph_tid_t* ptid)
 
   if (!in->flushing_caps) {
     ldout(cct, 10) << "mark_caps_flushing " << ccap_string(flushing) << " " << *in << dendl;
-    in->flushing_cap_seq = ++last_flush_seq;
     num_flushing_caps++;
   } else {
     ldout(cct, 10) << "mark_caps_flushing (more) " << ccap_string(flushing) << " " << *in << dendl;
@@ -3798,20 +3797,21 @@ void Client::wait_sync_caps(Inode *in, ceph_tid_t want)
   }
 }
 
-void Client::wait_sync_caps(uint64_t want)
+void Client::wait_sync_caps(ceph_tid_t want)
 {
  retry:
-  ldout(cct, 10) << "wait_sync_caps want " << want << " (last is " << last_flush_seq << ", "
+  ldout(cct, 10) << "wait_sync_caps want " << want  << " (last is " << last_flush_tid << ", "
           << num_flushing_caps << " total flushing)" << dendl;
   for (map<mds_rank_t,MetaSession*>::iterator p = mds_sessions.begin();
        p != mds_sessions.end();
        ++p) {
-    if (p->second->flushing_caps.empty())
+    MetaSession *s = p->second;
+    if (s->flushing_caps_tids.empty())
        continue;
-    Inode *in = p->second->flushing_caps.front();
-    if (in->flushing_cap_seq <= want) {
-      ldout(cct, 10) << " waiting on mds." << p->first << " tid " << in->flushing_cap_seq
-              << " (want " << want << ")" << dendl;
+    ceph_tid_t oldest_tid = *s->flushing_caps_tids.begin();
+    if (oldest_tid <= want) {
+      ldout(cct, 10) << " waiting on mds." << p->first << " tid " << oldest_tid
+                    << " (want " << want << ")" << dendl;
       sync_cond.Wait(client_lock);
       goto retry;
     }
@@ -4353,8 +4353,12 @@ void Client::handle_cap_flush_ack(MetaSession *session, Inode *in, Cap *cap, MCl
          << " cleaned " << ccap_string(cleaned) << " on " << *in
          << " with " << ccap_string(dirty) << dendl;
 
-  if (flushed)
+  if (flushed) {
     signal_cond_list(in->waitfor_caps);
+    if (session->flushing_caps_tids.empty() ||
+       *session->flushing_caps_tids.begin() > flush_ack_tid)
+      sync_cond.Signal();
+  }
 
   if (!cleaned) {
     ldout(cct, 10) << " tid " << m->get_client_tid() << " != any cap bit tids" << dendl;
@@ -4367,7 +4371,6 @@ void Client::handle_cap_flush_ack(MetaSession *session, Inode *in, Cap *cap, MCl
        ldout(cct, 10) << " " << *in << " !flushing" << dendl;
        in->flushing_cap_item.remove_myself();
        num_flushing_caps--;
-       sync_cond.Signal();
       }
       if (!in->caps_dirty())
        put_inode(in);
@@ -4990,7 +4993,7 @@ void Client::unmount()
   }
 
   flush_caps();
-  wait_sync_caps(last_flush_seq);
+  wait_sync_caps(last_flush_tid);
 
   // empty lru cache
   lru.lru_set_max(0);
@@ -8393,7 +8396,7 @@ int Client::_sync_fs()
   
   // flush caps
   flush_caps();
-  wait_sync_caps(last_flush_seq);
+  wait_sync_caps(last_flush_tid);
 
   // flush file data
   // FIXME

diff --git a/src/client/Client.h b/src/client/Client.h
index e6c74db5a4b136c43646f3753850537a4e2b6537..e7b19b9ea01ac54b1da19c6d4a50f0254eab6df5 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -292,7 +292,6 @@ public:
   // mds requests
   ceph_tid_t last_tid;
   ceph_tid_t oldest_tid; // oldest incomplete mds request, excluding setfilelock requests
-  ceph_tid_t last_flush_seq;
   ceph_tid_t last_flush_tid;
   map<ceph_tid_t, MetaRequest*> mds_requests;
 
@@ -565,7 +564,7 @@ protected:
   void put_cap_ref(Inode *in, int cap);
   void flush_snaps(Inode *in, bool all_again=false, CapSnap *again=0);
   void wait_sync_caps(Inode *in, ceph_tid_t want);
-  void wait_sync_caps(uint64_t want);
+  void wait_sync_caps(ceph_tid_t want);
   void queue_cap_snap(Inode *in, SnapContext &old_snapc);
   void finish_cap_snap(Inode *in, CapSnap *capsnap, int used);
   void _flushed_cap_snap(Inode *in, snapid_t seq);

diff --git a/src/client/Inode.cc b/src/client/Inode.cc
index e81ebfee13b9a7d8b939049ca5d5f161c7a6a604..aa4e49f454979eba6168872d9d7fb31986398730 100644 (file)
--- a/src/client/Inode.cc
+++ b/src/client/Inode.cc
@@ -394,7 +394,6 @@ void Inode::dump(Formatter *f) const
   f->dump_stream("dirty_caps") << ccap_string(dirty_caps);
   if (flushing_caps) {
     f->dump_stream("flushings_caps") << ccap_string(flushing_caps);
-    f->dump_unsigned("flushing_cap_seq", flushing_cap_seq);
     f->open_object_section("flushing_cap_tid");
     for (map<ceph_tid_t, int>::const_iterator p = flushing_cap_tids.begin();
         p != flushing_cap_tids.end();

diff --git a/src/client/Inode.h b/src/client/Inode.h
index 7ed4958e86adbf18fb8bdfb90eae3ca21108ec50..8cf3e37e9a5ef0e7b3b67c58d7846e1400b3b254 100644 (file)
--- a/src/client/Inode.h
+++ b/src/client/Inode.h
@@ -233,7 +233,6 @@ struct Inode {
   map<mds_rank_t, Cap*> caps;            // mds -> Cap
   Cap *auth_cap;
   unsigned dirty_caps, flushing_caps;
-  uint64_t flushing_cap_seq;
   std::map<ceph_tid_t, int> flushing_cap_tids;
   int shared_gen, cache_gen;
   int snap_caps, snap_cap_refs;
@@ -303,7 +302,7 @@ struct Inode {
       flags(0),
       qtree(NULL),
       dir_hashed(false), dir_replicated(false), auth_cap(NULL),
-      dirty_caps(0), flushing_caps(0), flushing_cap_seq(0), shared_gen(0), cache_gen(0),
+      dirty_caps(0), flushing_caps(0), shared_gen(0), cache_gen(0),
       snap_caps(0), snap_cap_refs(0),
       cap_item(this), flushing_cap_item(this),
       snaprealm(0), snaprealm_item(this),