mds: rebuild the internal states that tracking pending snapflush

Signed-off-by: Yan, Zheng <zyan@redhat.com>

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 951ed32283995b2e1d4c4dedd240d14e77ca3ff8..83ad0ded3f3b599b01aa245105dc14c6d885993d 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -5325,8 +5325,8 @@ bool MDCache::process_imported_caps()
        Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
        assert(session);
        for (auto r = q->second.begin(); r != q->second.end(); ++r) {
-         add_reconnected_cap(in, q->first, inodeno_t(r->second.capinfo.snaprealm));
          Capability *cap = in->reconnect_cap(q->first, r->second, session);
+         add_reconnected_cap(in, q->first, r->second);
          if (r->first >= 0) {
            if (cap->get_last_seq() == 0) // don't increase mseq if cap already exists
              cap->inc_mseq();
@@ -5372,6 +5372,42 @@ void MDCache::check_realm_past_parents(SnapRealm *realm)
   }
 }
 
+void MDCache::rebuild_need_snapflush(CInode *head_in, SnapRealm *realm,
+                                    client_t client, snapid_t snap_follows)
+{
+  dout(10) << "rebuild_need_snapflush " << snap_follows << " on " << *head_in << dendl;
+
+  const set<snapid_t>& snaps = realm->get_snaps();
+  snapid_t follows = snap_follows;
+
+  while (true) {
+    CInode *in = pick_inode_snap(head_in, follows);
+    if (in == head_in)
+      break;
+    dout(10) << " need snapflush from client." << client << " on " << *in << dendl;
+
+    /* TODO: we can check the reconnected/flushing caps to find 
+     *       which locks need gathering */
+    for (int i = 0; i < num_cinode_locks; i++) {
+      int lockid = cinode_lock_info[i].lock;
+      SimpleLock *lock = in->get_lock(lockid);
+      assert(lock);
+      in->client_snap_caps[lockid].insert(client);
+      in->auth_pin(lock);
+      lock->set_state(LOCK_SNAP_SYNC);
+      lock->get_wrlock(true);
+    }
+
+    for (auto p = snaps.lower_bound(in->first);
+        p != snaps.end() && *p <= in->last;
+        ++p) {
+      head_in->add_need_snapflush(in, *p, client);
+    }
+
+    follows = in->last;
+  }
+}
+
 /*
  * choose lock states based on reconnected caps
  */
@@ -5385,6 +5421,9 @@ void MDCache::choose_lock_states_and_reconnect_caps()
        i != inode_map.end();
        ++i) {
     CInode *in = i->second;
+
+    if (in->last != CEPH_NOSNAP)
+      continue;
  
     if (in->is_auth() && !in->is_base() && in->inode.is_dirty_rstat())
       in->mark_dirty_rstat();
@@ -5400,30 +5439,37 @@ void MDCache::choose_lock_states_and_reconnect_caps()
 
     check_realm_past_parents(realm);
 
-    map<CInode*,map<client_t,inodeno_t> >::iterator p = reconnected_caps.find(in);
+    auto p = reconnected_caps.find(in);
     if (p != reconnected_caps.end()) {
-
+      bool missing_snap_parent = false;
       // also, make sure client's cap is in the correct snaprealm.
-      for (map<client_t,inodeno_t>::iterator q = p->second.begin();
-          q != p->second.end();
-          ++q) {
-       if (q->second == realm->inode->ino()) {
-         dout(15) << "  client." << q->first << " has correct realm " << q->second << dendl;
+      for (auto q = p->second.begin(); q != p->second.end(); ++q) {
+       if (q->second.snap_follows > 0 && q->second.snap_follows < in->first - 1) {
+         if (realm->have_past_parents_open()) {
+           rebuild_need_snapflush(in, realm, q->first, q->second.snap_follows);
+         } else {
+           missing_snap_parent = true;
+         }
+       }
+
+       if (q->second.realm_ino == realm->inode->ino()) {
+         dout(15) << "  client." << q->first << " has correct realm " << q->second.realm_ino << dendl;
        } else {
-         dout(15) << "  client." << q->first << " has wrong realm " << q->second
+         dout(15) << "  client." << q->first << " has wrong realm " << q->second.realm_ino
                   << " != " << realm->inode->ino() << dendl;
          if (realm->have_past_parents_open()) {
            // ok, include in a split message _now_.
            prepare_realm_split(realm, q->first, in->ino(), splits);
          } else {
            // send the split later.
-           missing_snap_parents[realm->inode][q->first].insert(in->ino());
+           missing_snap_parent = true;
          }
        }
       }
+      if (missing_snap_parent)
+       missing_snap_parents[realm->inode].insert(in);
     }
   }    
-  reconnected_caps.clear();
 
   send_snaps(splits);
 }
@@ -5647,21 +5693,26 @@ void MDCache::open_snap_parents()
   map<client_t,MClientSnap*> splits;
   MDSGatherBuilder gather(g_ceph_context);
 
-  map<CInode*,map<client_t,set<inodeno_t> > >::iterator p = missing_snap_parents.begin();
+  auto p = missing_snap_parents.begin();
   while (p != missing_snap_parents.end()) {
     CInode *in = p->first;
     assert(in->snaprealm);
     if (in->snaprealm->open_parents(gather.new_sub())) {
       dout(10) << " past parents now open on " << *in << dendl;
-      
-      // include in a (now safe) snap split?
-      for (map<client_t,set<inodeno_t> >::iterator q = p->second.begin();
-          q != p->second.end();
-          ++q)
-       for (set<inodeno_t>::iterator r = q->second.begin();
-            r != q->second.end();
-            ++r) 
-         prepare_realm_split(in->snaprealm, q->first, *r, splits);
+
+      for (CInode *child : p->second) {
+       auto q = reconnected_caps.find(child);
+       assert(q != reconnected_caps.end());
+       for (auto r = q->second.begin(); r != q->second.end(); ++r) {
+         if (r->second.snap_follows > 0 && r->second.snap_follows < in->first - 1) {
+           rebuild_need_snapflush(child, in->snaprealm, r->first, r->second.snap_follows);
+         }
+         // make sure client's cap is in the correct snaprealm.
+         if (r->second.realm_ino != in->ino()) {
+           prepare_realm_split(in->snaprealm, r->first, child->ino(), splits);
+         }
+       }
+      }
 
       missing_snap_parents.erase(p++);
 
@@ -5713,6 +5764,7 @@ void MDCache::open_snap_parents()
     assert(rejoin_done != NULL);
     rejoin_done->complete(0);
     rejoin_done = NULL;
+    reconnected_caps.clear();
   }
 }
 
@@ -6044,6 +6096,9 @@ void MDCache::identify_files_to_recover(vector<CInode*>& recover_q, vector<CInod
     if (!in->is_auth())
       continue;
 
+    if (in->last != CEPH_NOSNAP)
+      continue;
+
     // Only normal files need file size recovery
     if (!in->is_file()) {
       continue;

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 925356a6354771067cb509c061b383e481ff73fa..a6df57dfc287d4fd8a443ebaac5a26d0fa7f835c 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -561,11 +561,18 @@ public:
   }
 
   // [reconnect/rejoin caps]
-  map<CInode*,map<client_t, inodeno_t> >  reconnected_caps;   // inode -> client -> realmino
+  struct reconnected_cap_info_t {
+    inodeno_t realm_ino;
+    snapid_t snap_follows;
+    reconnected_cap_info_t() : realm_ino(0), snap_follows(0) {}
+  };
+  map<CInode*,map<client_t, reconnected_cap_info_t> >  reconnected_caps;   // inode -> client -> snap_follows,realmino
   map<inodeno_t,map<client_t, snapid_t> > reconnected_snaprealms;  // realmino -> client -> realmseq
 
-  void add_reconnected_cap(CInode *in, client_t client, inodeno_t realm) {
-    reconnected_caps[in][client] = realm;
+  void add_reconnected_cap(CInode *in, client_t client, const cap_reconnect_t& icr) {
+    reconnected_cap_info_t &info = reconnected_caps[in][client];
+    info.realm_ino = inodeno_t(icr.capinfo.snaprealm);
+    info.snap_follows = icr.snap_follows;
   }
   void add_reconnected_snaprealm(client_t client, inodeno_t ino, snapid_t seq) {
     reconnected_snaprealms[ino][client] = seq;
@@ -589,13 +596,15 @@ public:
 
   // cap imports.  delayed snap parent opens.
   //  realm inode -> client -> cap inodes needing to split to this realm
-  map<CInode*,map<client_t, set<inodeno_t> > > missing_snap_parents; 
+  map<CInode*,set<CInode*> > missing_snap_parents;
   map<client_t,set<CInode*> > delayed_imported_caps;
 
   void do_cap_import(Session *session, CInode *in, Capability *cap,
                     uint64_t p_cap_id, ceph_seq_t p_seq, ceph_seq_t p_mseq,
                     int peer, int p_flags);
   void do_delayed_cap_imports();
+  void rebuild_need_snapflush(CInode *head_in, SnapRealm *realm, client_t client,
+                             snapid_t snap_follows);
   void check_realm_past_parents(SnapRealm *realm);
   void open_snap_parents();
 

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 37c5afd319fd335a654b9b59f4332e4445c3b3af..5db717e0f43e6436271f43c1e4e39c941a72e403 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -818,7 +818,7 @@ void Server::handle_client_reconnect(MClientReconnect *m)
       dout(15) << "open cap realm " << inodeno_t(p->second.capinfo.snaprealm)
               << " on " << *in << dendl;
       in->reconnect_cap(from, p->second, session);
-      mdcache->add_reconnected_cap(in, from, inodeno_t(p->second.capinfo.snaprealm));
+      mdcache->add_reconnected_cap(in, from, p->second);
       recover_filelocks(in, p->second.flockbl, m->get_orig_source().num());
       continue;
     }