mds: rejoin remote wrlocks and frozen auth pin

Includes remote wrlocks and frozen authpin in cache rejoin strong message

Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index aac9cf30a47199da968a51f35452014d61a8f083..bce314284db43fdb1f3bb1e423a70983400408d4 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -1325,7 +1325,7 @@ void Locker::remote_wrlock_start(SimpleLock *lock, int target, MDRequest *mut)
 
   // send lock request
   if (!lock->is_waiter_for(SimpleLock::WAIT_REMOTEXLOCK)) {
-    mut->start_locking(lock);
+    mut->start_locking(lock, target);
     mut->more()->slaves.insert(target);
     MMDSSlaveRequest *r = new MMDSSlaveRequest(mut->reqid, mut->attempt,
                                               MMDSSlaveRequest::OP_WRLOCK);
@@ -1410,9 +1410,9 @@ bool Locker::xlock_start(SimpleLock *lock, MDRequest *mut)
     
     // send lock request
     if (!lock->is_waiter_for(SimpleLock::WAIT_REMOTEXLOCK)) {
-      mut->start_locking(lock);
       int auth = lock->get_parent()->authority().first;
       mut->more()->slaves.insert(auth);
+      mut->start_locking(lock, auth);
       MMDSSlaveRequest *r = new MMDSSlaveRequest(mut->reqid, mut->attempt,
                                                 MMDSSlaveRequest::OP_XLOCK);
       r->set_lock_type(lock->get_type());

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 3ba86b51f8a4b5505964f8ff7950cbe5e24e375a..41e90b429b01fa419d9f44b10e50d2167275bc11 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -2649,6 +2649,9 @@ void MDCache::handle_mds_failure(int who)
                 << " to recover" << dendl;
        p->second->clear_ambiguous_auth();
       }
+
+      if (p->second->locking && p->second->locking_target_mds == who)
+       p->second->finish_locking(p->second->locking);
     }
   }
 
@@ -3500,6 +3503,11 @@ void MDCache::rejoin_send_rejoins()
            rejoin->add_inode_authpin(vinodeno_t(i.ino, i.snapid), p->second->reqid, p->second->attempt);
          else
            rejoin->add_dentry_authpin(i.dirfrag, i.dname, i.snapid, p->second->reqid, p->second->attempt);
+
+         if (p->second->has_more() && p->second->more()->is_remote_frozen_authpin &&
+             p->second->more()->rename_inode == (*q))
+           rejoin->add_inode_frozen_authpin(vinodeno_t(i.ino, i.snapid),
+                                            p->second->reqid, p->second->attempt);
        }
       }
       // xlocks
@@ -3522,6 +3530,22 @@ void MDCache::rejoin_send_rejoins()
                                     p->second->reqid, p->second->attempt);
        }
       }
+      // remote wrlocks
+      for (map<SimpleLock*, int>::iterator q = p->second->remote_wrlocks.begin();
+          q != p->second->remote_wrlocks.end();
+          ++q) {
+       int who = q->second;
+       if (rejoins.count(who) == 0) continue;
+       MMDSCacheRejoin *rejoin = rejoins[who];
+
+       dout(15) << " " << *p->second << " wrlock on " << q->second
+                << " " << q->first->get_parent() << dendl;
+       MDSCacheObjectInfo i;
+       q->first->get_parent()->set_object_info(i);
+       assert(i.ino);
+       rejoin->add_inode_wrlock(vinodeno_t(i.ino, i.snapid), q->first->get_type(),
+                                p->second->reqid, p->second->attempt);
+      }
     }
   }
 
@@ -4215,7 +4239,9 @@ void MDCache::handle_cache_rejoin_strong(MMDSCacheRejoin *strong)
        dout(10) << " dn xlock by " << r << " on " << *dn << dendl;
        MDRequest *mdr = request_get(r.reqid);  // should have this from auth_pin above.
        assert(mdr->is_auth_pinned(dn));
-       dn->lock.set_state(LOCK_LOCK);
+       if (dn->lock.is_stable())
+         dn->auth_pin(&dn->lock);
+       dn->lock.set_state(LOCK_XLOCK);
        dn->lock.get_xlock(mdr, mdr->get_client());
        mdr->xlocks.insert(&dn->lock);
        mdr->locks.insert(&dn->lock);
@@ -4258,9 +4284,14 @@ void MDCache::handle_cache_rejoin_strong(MMDSCacheRejoin *strong)
              mdr = request_get(r.reqid);
            else
              mdr = request_start_slave(r.reqid, r.attempt, from);
+           if (strong->frozen_authpin_inodes.count(in->vino())) {
+             assert(!in->get_num_auth_pins());
+             mdr->freeze_auth_pin(in);
+           } else {
+             assert(!in->is_frozen_auth_pin());
+           }
            mdr->auth_pin(in);
          }
-         
          // xlock(s)?
          if (strong->xlocked_inodes.count(in->vino())) {
            for (map<int,MMDSCacheRejoin::slave_reqid>::iterator r = strong->xlocked_inodes[in->vino()].begin();
@@ -4270,7 +4301,9 @@ void MDCache::handle_cache_rejoin_strong(MMDSCacheRejoin *strong)
              dout(10) << " inode xlock by " << r->second << " on " << *lock << " on " << *in << dendl;
              MDRequest *mdr = request_get(r->second.reqid);  // should have this from auth_pin above.
              assert(mdr->is_auth_pinned(in));
-             lock->set_state(LOCK_LOCK);
+             if (lock->is_stable())
+               in->auth_pin(lock);
+             lock->set_state(LOCK_XLOCK);
              if (lock == &in->filelock)
                in->loner_cap = -1;
              lock->get_xlock(mdr, mdr->get_client());
@@ -4278,6 +4311,23 @@ void MDCache::handle_cache_rejoin_strong(MMDSCacheRejoin *strong)
              mdr->locks.insert(lock);
            }
          }
+         // wrlock(s)?
+         if (strong->wrlocked_inodes.count(in->vino())) {
+           for (map<int,MMDSCacheRejoin::slave_reqid>::iterator r = strong->wrlocked_inodes[in->vino()].begin();
+                r != strong->wrlocked_inodes[in->vino()].end();
+                ++r) {
+             SimpleLock *lock = in->get_lock(r->first);
+             dout(10) << " inode wrlock by " << r->second << " on " << *lock << " on " << *in << dendl;
+             MDRequest *mdr = request_get(r->second.reqid);  // should have this from auth_pin above.
+             assert(mdr->is_auth_pinned(in));
+             lock->set_state(LOCK_LOCK);
+             if (lock == &in->filelock)
+               in->loner_cap = -1;
+             lock->get_wrlock(true);
+             mdr->wrlocks.insert(lock);
+             mdr->locks.insert(lock);
+           }
+         }
        } else {
          dout(10) << " sender has dentry but not inode, adding them as a replica" << dendl;
        }

diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc
index 367181cc930b690a63274fc28b9b1031bafea7a4..62968f77f8ab7c63fab03095889c86fe716c21f5 100644 (file)
--- a/src/mds/Mutation.cc
+++ b/src/mds/Mutation.cc
@@ -47,17 +47,19 @@ void Mutation::drop_pins()
   pins.clear();
 }
 
-void Mutation::start_locking(SimpleLock *lock)
+void Mutation::start_locking(SimpleLock *lock, int target)
 {
   assert(locking == NULL);
   pin(lock->get_parent());
   locking = lock;
+  locking_target_mds = target;
 }
 
 void Mutation::finish_locking(SimpleLock *lock)
 {
   assert(locking == lock);
   locking = NULL;
+  locking_target_mds = -1;
 }
 
 
@@ -228,6 +230,13 @@ void MDRequest::unfreeze_auth_pin()
   more()->is_freeze_authpin = false;
 }
 
+void MDRequest::set_remote_frozen_auth_pin(CInode *inode)
+{
+  assert(!more()->rename_inode || more()->rename_inode == inode);
+  more()->rename_inode = inode;
+  more()->is_remote_frozen_authpin = true;
+}
+
 void MDRequest::set_ambiguous_auth(CInode *inode)
 {
   assert(!more()->rename_inode || more()->rename_inode == inode);

diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index c64657ffee9abf5fdfa1e62d525175b8160d6e45..bb5f1f6febc690fc87b7514721c25b64dc82d99e 100644 (file)
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -61,6 +61,7 @@ struct Mutation {
   // lock we are currently trying to acquire.  if we give up for some reason,
   // be sure to eval() this.
   SimpleLock *locking;
+  int locking_target_mds;
 
   // if this flag is set, do not attempt to acquire further locks.
   //  (useful for wrlock, which may be a moving auth target)
@@ -82,12 +83,14 @@ struct Mutation {
       ls(0),
       slave_to_mds(-1),
       locking(NULL),
+      locking_target_mds(-1),
       done_locking(false), committing(false), aborted(false), killed(false) { }
   Mutation(metareqid_t ri, __u32 att=0, int slave_to=-1)
     : reqid(ri), attempt(att),
       ls(0),
       slave_to_mds(slave_to), 
       locking(NULL),
+      locking_target_mds(-1),
       done_locking(false), committing(false), aborted(false), killed(false) { }
   virtual ~Mutation() {
     assert(locking == NULL);
@@ -113,7 +116,7 @@ struct Mutation {
   void set_stickydirs(CInode *in);
   void drop_pins();
 
-  void start_locking(SimpleLock *lock);
+  void start_locking(SimpleLock *lock, int target=-1);
   void finish_locking(SimpleLock *lock);
 
   // auth pins
@@ -204,6 +207,7 @@ struct MDRequest : public Mutation {
     CInode* rename_inode;
     bool is_freeze_authpin;
     bool is_ambiguous_auth;
+    bool is_remote_frozen_authpin;
     bool is_inode_exporter;
 
     map<client_t,entity_inst_t> imported_client_map;
@@ -224,8 +228,8 @@ struct MDRequest : public Mutation {
     More() : 
       src_reanchor_atid(0), dst_reanchor_atid(0), inode_import_v(0),
       rename_inode(0), is_freeze_authpin(false), is_ambiguous_auth(false),
-      is_inode_exporter(false), flock_was_waiting(false),
-      stid(0), slave_commit(0) { }
+      is_remote_frozen_authpin(false), is_inode_exporter(false),
+      flock_was_waiting(false), stid(0), slave_commit(0) { }
   } *_more;
 
 
@@ -280,6 +284,7 @@ struct MDRequest : public Mutation {
   bool did_ino_allocation();
   bool freeze_auth_pin(CInode *inode);
   void unfreeze_auth_pin();
+  void set_remote_frozen_auth_pin(CInode *inode);
   bool can_auth_pin(MDSCacheObject *object);
   void drop_local_auth_pins();
   void set_ambiguous_auth(CInode *inode);

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 6d685be1e56814da1b9ab4ff7ee572105b8febb5..9d08c202c67a1ee535d8c1b31a8ecc4a99c8aa26 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1520,7 +1520,7 @@ void Server::handle_slave_auth_pin(MDRequest *mdr)
 
     objects.push_back(object);
     if (*p == mdr->slave_request->get_authpin_freeze())
-      auth_pin_freeze = dynamic_cast<CInode*>(object);
+      auth_pin_freeze = (CInode*)object;
   }
   
   // can we auth pin them?
@@ -1583,6 +1583,9 @@ void Server::handle_slave_auth_pin(MDRequest *mdr)
     reply->get_authpins().push_back(info);
   }
 
+  if (auth_pin_freeze)
+    auth_pin_freeze->set_object_info(reply->get_authpin_freeze());
+
   mds->send_message_mds(reply, mdr->slave_to_mds);
   
   // clean up this request
@@ -1607,6 +1610,8 @@ void Server::handle_slave_auth_pin_ack(MDRequest *mdr, MMDSSlaveRequest *ack)
     dout(10) << " remote has pinned " << *object << dendl;
     if (!mdr->is_auth_pinned(object))
       mdr->remote_auth_pins.insert(object);
+    if (*p == ack->get_authpin_freeze())
+      mdr->set_remote_frozen_auth_pin((CInode *)object);
     pinned.insert(object);
   }
 
@@ -5542,6 +5547,8 @@ void Server::handle_client_rename(MDRequest *mdr)
     le->had_slaves = true;
     
     mds->mdcache->add_uncommitted_master(mdr->reqid, mdr->ls, mdr->more()->witnessed);
+    // no need to send frozen auth pin to recovring auth MDS of srci
+    mdr->more()->is_remote_frozen_authpin = false;
   }
   
   _rename_prepare(mdr, &le->metablob, &le->client_map, srcdn, destdn, straydn);

diff --git a/src/messages/MMDSCacheRejoin.h b/src/messages/MMDSCacheRejoin.h
index e5a86ee45cb0c75a86982c52efe9deacf97371cb..825400d6eeaebe4f36e87f58e46c53d6fddd1c50 100644 (file)
--- a/src/messages/MMDSCacheRejoin.h
+++ b/src/messages/MMDSCacheRejoin.h
@@ -184,7 +184,9 @@ class MMDSCacheRejoin : public Message {
     }
   };
   map<vinodeno_t, slave_reqid> authpinned_inodes;
+  map<vinodeno_t, slave_reqid> frozen_authpin_inodes;
   map<vinodeno_t, map<__s32, slave_reqid> > xlocked_inodes;
+  map<vinodeno_t, map<__s32, slave_reqid> > wrlocked_inodes;
   map<dirfrag_t, map<string_snap_t, slave_reqid> > authpinned_dentries;
   map<dirfrag_t, map<string_snap_t, slave_reqid> > xlocked_dentries;
   
@@ -227,9 +229,15 @@ public:
   void add_inode_authpin(vinodeno_t ino, const metareqid_t& ri, __u32 attempt) {
     authpinned_inodes[ino] = slave_reqid(ri, attempt);
   }
+  void add_inode_frozen_authpin(vinodeno_t ino, const metareqid_t& ri, __u32 attempt) {
+    frozen_authpin_inodes[ino] = slave_reqid(ri, attempt);
+  }
   void add_inode_xlock(vinodeno_t ino, int lt, const metareqid_t& ri, __u32 attempt) {
     xlocked_inodes[ino][lt] = slave_reqid(ri, attempt);
   }
+  void add_inode_wrlock(vinodeno_t ino, int lt, const metareqid_t& ri, __u32 attempt) {
+    wrlocked_inodes[ino][lt] = slave_reqid(ri, attempt);
+  }
 
   void add_scatterlock_state(CInode *in) {
     if (inode_scatterlocks.count(in->ino()))
@@ -278,7 +286,9 @@ public:
     ::encode(inode_locks, payload);
     ::encode(inode_scatterlocks, payload);
     ::encode(authpinned_inodes, payload);
+    ::encode(frozen_authpin_inodes, payload);
     ::encode(xlocked_inodes, payload);
+    ::encode(wrlocked_inodes, payload);
     ::encode(cap_export_bl, payload);
     ::encode(strong_dirfrags, payload);
     ::encode(weak, payload);
@@ -296,7 +306,9 @@ public:
     ::decode(inode_locks, p);
     ::decode(inode_scatterlocks, p);
     ::decode(authpinned_inodes, p);
+    ::decode(frozen_authpin_inodes, p);
     ::decode(xlocked_inodes, p);
+    ::decode(wrlocked_inodes, p);
     ::decode(cap_export_bl, p);
     if (cap_export_bl.length()) {
       bufferlist::iterator q = cap_export_bl.begin();