mds: reconnect caps added via replay+issue_new_caps

Also, set up the initial lock states after reconnecting them!

diff --git a/src/TODO b/src/TODO
index 2e9ad455f0f8a61736e6d188c4616090cc07185a..e7ca7821f35db8790a90223e39dc41bde11a796f 100644 (file)
--- a/src/TODO
+++ b/src/TODO
@@ -129,8 +129,6 @@ uclient
 - fix readdir vs fragment race by keeping a separate frag pos, and ignoring dentries below it
 
 mds
-- fix file recovery
-  - disallow access until file is recovered?
 - add an up:shadow mode?
   - tail the mds log as it is written
   - periodically check head so that we trim, too

diff --git a/src/include/atomic.h b/src/include/atomic.h
index d744b4052c8446df653a24196e118628fd8d3729..9cd3f785dec0f6ab027b305aa0603c8285affafc 100644 (file)
--- a/src/include/atomic.h
+++ b/src/include/atomic.h
@@ -38,7 +38,7 @@ public:
 
 #else
 /*
- * crappy slow implementation that uses a pthreads mutex.
+ * crappy slow implementation that uses a pthreads spinlock.
  */
 #include "common/Spinlock.h"
 

diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index 5db6bcc6c2b13f28808dfc85aeb8a4af107811ea..cdfe015be1c619d21fa0cdcf8c1443df0dea060b 100644 (file)
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -602,6 +602,15 @@ public:
        lock->set_state(LOCK_SYNC);  // might have been lock, previously
     }
   }
+  void choose_lock_states() {
+    int issued = get_caps_issued();
+    if (is_auth() && (issued & CEPH_CAP_ANY_EXCL))
+      try_choose_loner();
+    choose_lock_state(&filelock, issued);
+    choose_lock_state(&authlock, issued);
+    choose_lock_state(&xattrlock, issued);
+    choose_lock_state(&linklock, issued);
+  }
 
   int count_nonstale_caps() {
     int n = 0;
@@ -665,7 +674,7 @@ public:
     } else {
       cap = add_client_cap(client, session);
       cap->set_wanted(icr.wanted);
-      cap->issue(icr.issued);
+      cap->issue_norevoke(icr.issued);
     }
     cap->set_cap_id(icr.cap_id);
     cap->set_last_issue_stamp(g_clock.recent_now());

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 439a0a4274b339675ad8518e09be0ad536b153a1..b6cb4fd79fc7962f97f63fb1e2cc7911d477fbbb 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -1047,11 +1047,18 @@ void Locker::file_update_finish(CInode *in, Mutation *mut, bool share, int clien
 Capability* Locker::issue_new_caps(CInode *in,
                                   int mode,
                                   Session *session,
-                                  SnapRealm *realm)
+                                  SnapRealm *realm,
+                                  bool is_replay)
 {
   dout(7) << "issue_new_caps for mode " << mode << " on " << *in << dendl;
   bool is_new;
 
+  // if replay, try to reconnect cap, and otherwise do nothing.
+  if (is_replay) {
+    mds->mdcache->try_reconnect_cap(in, session);
+    return 0;
+  }
+
   // my needs
   assert(session->inst.name.is_client());
   int my_client = session->inst.name.num();

diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index a836984d21c0d1a2a531a26dfcb3fe5a2625adc3..3b9d475e23d0a4aa553232f9dadd9b4b01ba159b 100644 (file)
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -208,7 +208,7 @@ public:
   // -- file i/o --
  public:
   version_t issue_file_data_version(CInode *in);
-  Capability* issue_new_caps(CInode *in, int mode, Session *session, SnapRealm *conrealm=0);
+  Capability* issue_new_caps(CInode *in, int mode, Session *session, SnapRealm *conrealm, bool is_replay);
   bool issue_caps(CInode *in, Capability *only_cap=0);
   void issue_truncate(CInode *in);
   void revoke_stale_caps(Session *session);

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index bcaf91f67c3994feea47b3e373506146412f7406..476895cea164eb007c1df35bfc657b0834361e46 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -3819,16 +3819,8 @@ void MDCache::process_reconnected_caps()
     CInode *in = p->first;
     p++;
 
-    int issued = in->get_caps_issued();
-    if (in->is_auth() &&
-       (issued & CEPH_CAP_ANY_EXCL))
-      in->try_choose_loner();
-    in->choose_lock_state(&in->filelock, issued);
-    in->choose_lock_state(&in->authlock, issued);
-    in->choose_lock_state(&in->xattrlock, issued);
-    in->choose_lock_state(&in->linklock, issued);
-    dout(15) << " issued " << ccap_string(issued)
-            << " chose lock states on " << *in << dendl;
+    in->choose_lock_states();
+    dout(15) << " chose lock states on " << *in << dendl;
 
     SnapRealm *realm = in->find_snaprealm();
 
@@ -3954,6 +3946,24 @@ void MDCache::rejoin_import_cap(CInode *in, int client, ceph_mds_cap_reconnect&
     do_cap_import(session, in, cap);
 }
 
+void MDCache::try_reconnect_cap(CInode *in, Session *session)
+{
+  int client = session->get_client();
+  ceph_mds_cap_reconnect *rc = get_replay_cap_reconnect(in->ino(), client);
+  if (rc) {
+    in->reconnect_cap(client, *rc, session);
+    dout(10) << "try_reconnect_cap client" << client
+            << " reconnect wanted " << ccap_string(rc->wanted)
+            << " issue " << ccap_string(rc->issued)
+            << " on " << *in << dendl;
+    remove_replay_cap_reconnect(in->ino(), client);
+
+    in->choose_lock_states();
+    dout(15) << " chose lock states on " << *in << dendl;
+  }
+}
+
+
 
 // -------
 // cap imports and delayed snap parent opens

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 1ad4aead0f60159161956f2b9291c7d0b66fb562..ab2d17da9d9f925753698f3cebc244644a8250e5 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -32,6 +32,7 @@
 class Logger;
 
 class MDS;
+class Session;
 class Migrator;
 class Renamer;
 
@@ -746,6 +747,7 @@ public:
   void send_snaps(map<int,MClientSnap*>& splits);
   void rejoin_import_cap(CInode *in, int client, ceph_mds_cap_reconnect& icr, int frommds);
   void finish_snaprealm_reconnect(int client, SnapRealm *realm, snapid_t seq);
+  void try_reconnect_cap(CInode *in, Session *session);
 
   // cap imports.  delayed snap parent opens.
   //  realm inode -> client -> cap inodes needing to split to this realm

diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 7f1f1e006cbb383162685d73591bef12d775dd94..5ddee81adf3dabbb425c016bc60711a911945921 100644 (file)
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -1295,19 +1295,17 @@ bool MDS::_dispatch(Message *m)
 
   // finish any triggered contexts
   static bool finishing = false;
-  if (!finishing && finished_queue.size()) {
-    dout(7) << "mds has " << finished_queue.size() << " queued contexts" << dendl;
-    dout(10) << finished_queue << dendl;
-    list<Context*> ls;
-    ls.splice(ls.begin(), finished_queue);
-    assert(finished_queue.empty());
-    finishing = true;
-    finish_contexts(ls);
-    finishing = false;
-  } else {
+  if (!finishing) {
+    while (finished_queue.size()) {
+      dout(7) << "mds has " << finished_queue.size() << " queued contexts" << dendl;
+      dout(10) << finished_queue << dendl;
+      finishing = true;
+      finish_contexts(finished_queue);
+      finishing = false;
+    }
+
     // done with all client replayed requests?
-    if (!finishing &&
-       is_clientreplay() &&
+    if (is_clientreplay() &&
        mdcache->is_open() &&
        waiting_for_replay.empty() &&
        want_state == MDSMap::STATE_CLIENTREPLAY) {

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 83d0a0746cebec7c911374669104e8d939b79b32..df4fcec4aa1080a230744f81c29a7cd0e79185d3 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -697,7 +697,6 @@ void Server::early_reply(MDRequest *mdr, CInode *tracei, CDentry *tracedn)
 void Server::reply_request(MDRequest *mdr, MClientReply *reply, CInode *tracei, CDentry *tracedn) 
 {
   MClientRequest *req = mdr->client_request;
-  int client = mdr->get_client();
   
   dout(10) << "reply_request " << reply->get_result() 
           << " (" << strerror(-reply->get_result())
@@ -760,21 +759,8 @@ void Server::reply_request(MDRequest *mdr, MClientReply *reply, CInode *tracei,
     if (!did_early_reply &&   // don't issue leases if we sent an earlier reply already
        (tracei || tracedn)) {
       if (is_replay) {
-       // replay ops don't get a trace.
-       // reconnect cap to created inode?
-       if (tracei) {
-         ceph_mds_cap_reconnect *rc = mdcache->get_replay_cap_reconnect(tracei->ino(), client);
-         if (rc) {
-           Capability *cap = tracei->get_client_cap(client);
-           // we should only have the cap reconnect for ONE client, and from ourselves.
-           dout(10) << " incorporating cap reconnect wanted " << ccap_string(rc->wanted)
-                    << " issue " << ccap_string(rc->issued) << " on " << *tracei << dendl;
-           cap->set_wanted(rc->wanted);
-           cap->issue_norevoke(rc->issued);
-           cap->set_cap_id(rc->cap_id);
-           mdcache->remove_replay_cap_reconnect(tracei->ino(), client);
-         }
-       }
+       if (tracei)
+         mdcache->try_reconnect_cap(tracei, session);
       } else {
        // include metadata in reply
        set_trace_dist(session, reply, tracei, tracedn, snapid, dentry_wanted);
@@ -1959,10 +1945,11 @@ void Server::handle_client_open(MDRequest *mdr)
   if (cur->is_file() || cur->is_dir()) {
     if (mdr->snapid == CEPH_NOSNAP) {
       // register new cap
-      Capability *cap = mds->locker->issue_new_caps(cur, cmode, mdr->session);
-      dout(12) << "open issued caps " << ccap_string(cap->pending())
-              << " for " << req->get_orig_source()
-              << " on " << *cur << dendl;
+      Capability *cap = mds->locker->issue_new_caps(cur, cmode, mdr->session, 0, req->is_replay());
+      if (cap)
+       dout(12) << "open issued caps " << ccap_string(cap->pending())
+                << " for " << req->get_orig_source()
+                << " on " << *cur << dendl;
     } else {
       int caps = ceph_caps_for_mode(cmode);
       dout(12) << "open issued IMMUTABLE SNAP caps " << ccap_string(caps)
@@ -2095,7 +2082,7 @@ void Server::handle_client_openc(MDRequest *mdr)
   le->metablob.add_primary_dentry(dn, true, in);
 
   // do the open
-  mds->locker->issue_new_caps(in, cmode, mdr->session, realm);
+  mds->locker->issue_new_caps(in, cmode, mdr->session, realm, req->is_replay());
   in->authlock.set_state(LOCK_EXCL);
   in->xattrlock.set_state(LOCK_EXCL);
 
@@ -2442,7 +2429,7 @@ void Server::handle_client_opent(MDRequest *mdr, int cmode)
   
   // do the open
   SnapRealm *realm = in->find_snaprealm();
-  mds->locker->issue_new_caps(in, cmode, mdr->session, realm);
+  mds->locker->issue_new_caps(in, cmode, mdr->session, realm, mdr->client_request->is_replay());
 
   // make sure ino gets into the journal
   le->metablob.add_opened_ino(in->ino());
@@ -2763,15 +2750,17 @@ void Server::handle_client_mkdir(MDRequest *mdr)
   
   // issue a cap on the directory
   int cmode = CEPH_FILE_MODE_RDWR;
-  Capability *cap = mds->locker->issue_new_caps(newi, cmode, mdr->session, realm);
-  cap->set_wanted(0);
-
-  // put locks in excl mode
-  newi->filelock.set_state(LOCK_EXCL);
-  newi->authlock.set_state(LOCK_EXCL);
-  newi->xattrlock.set_state(LOCK_EXCL);
-  cap->issue_norevoke(CEPH_CAP_AUTH_EXCL|CEPH_CAP_AUTH_SHARED|
-                     CEPH_CAP_XATTR_EXCL|CEPH_CAP_XATTR_SHARED);
+  Capability *cap = mds->locker->issue_new_caps(newi, cmode, mdr->session, realm, req->is_replay());
+  if (cap) {
+    cap->set_wanted(0);
+
+    // put locks in excl mode
+    newi->filelock.set_state(LOCK_EXCL);
+    newi->authlock.set_state(LOCK_EXCL);
+    newi->xattrlock.set_state(LOCK_EXCL);
+    cap->issue_norevoke(CEPH_CAP_AUTH_EXCL|CEPH_CAP_AUTH_SHARED|
+                       CEPH_CAP_XATTR_EXCL|CEPH_CAP_XATTR_SHARED);
+  }
 
   // make sure this inode gets into the journal
   le->metablob.add_opened_ino(newi->ino());

diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
index 12516e3daf8de82dd84707d32c8855b486f2e965..9310fa01e999218d55518e3f8c18cab18c9246ba 100644 (file)
--- a/src/mon/MDSMonitor.cc
+++ b/src/mon/MDSMonitor.cc
@@ -357,6 +357,8 @@ void MDSMonitor::committed()
       mdsmap.is_stopped()) 
     mon->messenger->send_message(new MGenericMessage(CEPH_MSG_SHUTDOWN), 
                                 mon->monmap->get_inst(mon->whoami));
+
+  tick();
 }