client: fix dirty caps might never be flushed

Fixes: http://tracker.ceph.com/issues/22546
Signed-off-by: dongdong tao <tdd21151186@gmail.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 1c2d66002eab9cae54558fdf7aaff549c9239576..41ccab26f6aa173103acd0468b2da212efcc2c9f 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -3203,7 +3203,7 @@ void Client::cap_delay_requeue(Inode *in)
   ldout(cct, 10) << "cap_delay_requeue on " << *in << dendl;
   in->hold_caps_until = ceph_clock_now();
   in->hold_caps_until += cct->_conf->client_caps_release_delay;
-  delayed_caps.push_back(&in->cap_item);
+  delayed_list.push_back(&in->delay_cap_item);
 }
 
 void Client::send_cap(Inode *in, MetaSession *session, Cap *cap,
@@ -3842,10 +3842,7 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id
   mds_rank_t mds = mds_session->mds_num;
   const auto &capem = in->caps.emplace(std::piecewise_construct, std::forward_as_tuple(mds), std::forward_as_tuple(in, mds_session));
   Cap &cap = capem.first->second;
-  if (capem.second) {
-    /* new cap inserted */
-    cap_list.push_back(&in->cap_item);
-  } else {
+  if (!capem.second) {
     /*
      * auth mds of the inode changed. we received the cap export
      * message, but still haven't received the cap import message.
@@ -3978,7 +3975,7 @@ void Client::remove_session_caps(MetaSession *s)
        in->flushing_cap_tids.clear();
       }
       in->flushing_caps = 0;
-      in->dirty_caps = 0;
+      mark_caps_clean(in);
       put_inode(in);
     }
   }
@@ -4145,11 +4142,19 @@ void Client::force_session_readonly(MetaSession *s)
 
 void Client::mark_caps_dirty(Inode *in, int caps)
 {
-  ldout(cct, 10) << "mark_caps_dirty " << *in << " " << ccap_string(in->dirty_caps) << " -> "
+  ldout(cct, 10) << __func__ << " " << *in << " " << ccap_string(in->dirty_caps) << " -> "
           << ccap_string(in->dirty_caps | caps) << dendl;
   if (caps && !in->caps_dirty())
     in->get();
   in->dirty_caps |= caps;
+  dirty_list.push_back(&in->dirty_cap_item);
+}
+
+void Client::mark_caps_clean(Inode *in)
+{
+  ldout(cct, 10) << __func__ << " " << *in << dendl;
+  in->dirty_caps = 0;
+  in->dirty_cap_item.remove_myself(); 
 }
 
 int Client::mark_caps_flushing(Inode *in, ceph_tid_t* ptid)
@@ -4170,7 +4175,7 @@ int Client::mark_caps_flushing(Inode *in, ceph_tid_t* ptid)
   }
 
   in->flushing_caps |= flushing;
-  in->dirty_caps = 0;
+  mark_caps_clean(in);
  
   if (!in->flushing_cap_item.is_on_list())
     session->flushing_caps.push_back(&in->flushing_cap_item);
@@ -4206,20 +4211,20 @@ void Client::adjust_session_flushing_caps(Inode *in, MetaSession *old_s,  MetaSe
 void Client::flush_caps_sync()
 {
   ldout(cct, 10) << __func__ << dendl;
-  xlist<Inode*>::iterator p = delayed_caps.begin();
+  xlist<Inode*>::iterator p = delayed_list.begin();
   while (!p.end()) {
     unsigned flags = CHECK_CAPS_NODELAY;
     Inode *in = *p;
 
     ++p;
-    delayed_caps.pop_front();
-    if (p.end() && cap_list.empty())
+    delayed_list.pop_front();
+    if (p.end() && dirty_list.empty())
       flags |= CHECK_CAPS_SYNCHRONOUS;
     check_caps(in, flags);
   }
 
   // other caps, too
-  p = cap_list.begin();
+  p = dirty_list.begin();
   while (!p.end()) {
     unsigned flags = CHECK_CAPS_NODELAY;
     Inode *in = *p;
@@ -6029,14 +6034,13 @@ void Client::tick()
   }
 
   // delayed caps
-  xlist<Inode*>::iterator p = delayed_caps.begin();
+  xlist<Inode*>::iterator p = delayed_list.begin();
   while (!p.end()) {
     Inode *in = *p;
     ++p;
     if (in->hold_caps_until > now)
       break;
-    delayed_caps.pop_front();
-    cap_list.push_back(&in->cap_item);
+    delayed_list.pop_front();
     check_caps(in, CHECK_CAPS_NODELAY);
   }
 

diff --git a/src/client/Client.h b/src/client/Client.h
index a43b31c919384c627f422ca565641a4c1c23f30f..2a353f4b546b56cc6cfa3ddbcbcd6ae0bbd72f78 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -428,8 +428,8 @@ protected:
   Inode*                 root_ancestor;
   LRU                    lru;    // lru list of Dentry's in our local metadata cache.
 
-  // all inodes with caps sit on either cap_list or delayed_caps.
-  xlist<Inode*> delayed_caps, cap_list;
+  // dirty_list keeps all the dirty inodes before flushing.
+  xlist<Inode*> delayed_list, dirty_list;
   int num_flushing_caps;
   ceph::unordered_map<inodeno_t,SnapRealm*> snap_realms;
 
@@ -633,6 +633,7 @@ protected:
   void remove_all_caps(Inode *in);
   void remove_session_caps(MetaSession *session);
   void mark_caps_dirty(Inode *in, int caps);
+  void mark_caps_clean(Inode *in);
   int mark_caps_flushing(Inode *in, ceph_tid_t *ptid);
   void adjust_session_flushing_caps(Inode *in, MetaSession *old_s, MetaSession *new_s);
   void flush_caps_sync();

diff --git a/src/client/Inode.cc b/src/client/Inode.cc
index 86dbd030db079090a2247febce6aa19d803ddfda..8029abd530be7daca3b19d39bb59b8b0c5eebb92 100644 (file)
--- a/src/client/Inode.cc
+++ b/src/client/Inode.cc
@@ -14,7 +14,8 @@
 
 Inode::~Inode()
 {
-  cap_item.remove_myself();
+  delay_cap_item.remove_myself();
+  dirty_cap_item.remove_myself(); 
   snaprealm_item.remove_myself();
 
   if (snapdir_parent) {

diff --git a/src/client/Inode.h b/src/client/Inode.h
index c5fbb8ff9ec134996c09ac341565bc86c9d81aca..751ba6c55e93b1a28bea44c1925216f2984499da 100644 (file)
--- a/src/client/Inode.h
+++ b/src/client/Inode.h
@@ -198,7 +198,7 @@ struct Inode {
   int shared_gen, cache_gen;
   int snap_caps, snap_cap_refs;
   utime_t hold_caps_until;
-  xlist<Inode*>::item cap_item, flushing_cap_item;
+  xlist<Inode*>::item delay_cap_item, dirty_cap_item, flushing_cap_item;
 
   SnapRealm *snaprealm;
   xlist<Inode*>::item snaprealm_item;
@@ -268,7 +268,7 @@ struct Inode {
       cap_dirtier_uid(-1), cap_dirtier_gid(-1),
       dirty_caps(0), flushing_caps(0), shared_gen(0), cache_gen(0),
       snap_caps(0), snap_cap_refs(0),
-      cap_item(this), flushing_cap_item(this),
+      delay_cap_item(this), dirty_cap_item(this), flushing_cap_item(this),
       snaprealm(0), snaprealm_item(this),
       oset((void *)this, newlayout->pool_id, this->ino),
       reported_size(0), wanted_max_size(0), requested_max_size(0),