mds: track caps we release in requests, and reissue on request cleanup if needed

We may suppress cap issue when we release caps in a request.  A lock state
change may have changed the allowed caps and should normally have issued
caps, but we suppress that because the request will way to embed that
normally.  When we cleanup the request, if we haven't issued on that cap
for some reason (maybe the request errored out, or forwarded, etc.), force
an issue then.

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index b527c499e05881fc076946a8fd7960fa989afa8e..1c7c95e40b5a029668723a66492ad84c1f1426f1 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -1769,7 +1769,8 @@ void Locker::handle_client_caps(MClientCaps *m)
   delete m;
 }
 
-void Locker::process_cap_update(int client, inodeno_t ino, __u64 cap_id, int caps, int wanted,
+void Locker::process_cap_update(MDRequest *mdr, int client,
+                               inodeno_t ino, __u64 cap_id, int caps, int wanted,
                                int seq, int issue_seq, int mseq,
                                const nstring& dname)
 {
@@ -1791,6 +1792,9 @@ void Locker::process_cap_update(int client, inodeno_t ino, __u64 cap_id, int cap
     cap->inc_suppress();
     eval(in, CEPH_CAP_LOCKS);
     cap->dec_suppress();
+
+    // take note; we may need to reissue on this cap later
+    mdr->cap_releases[in->vino()] = cap->get_last_seq();
   }
   
   if (dname.length()) {
@@ -1805,7 +1809,26 @@ void Locker::process_cap_update(int client, inodeno_t ino, __u64 cap_id, int cap
       }
     }
   }
+}
 
+void Locker::kick_cap_releases(MDRequest *mdr)
+{
+  int client = mdr->get_client();
+  for (map<vinodeno_t,ceph_seq_t>::iterator p = mdr->cap_releases.begin();
+       p != mdr->cap_releases.end();
+       p++) {
+    CInode *in = mdcache->get_inode(p->first);
+    if (!in)
+      continue;
+    Capability *cap = in->get_client_cap(client);
+    if (!cap)
+      continue;
+    if (cap->get_last_sent() == p->second) {
+      dout(10) << "kick_cap_releases released at current seq " << p->second
+              << ", reissuing" << dendl;
+      issue_caps(in, cap);
+    }
+  }
 }
 
 /*

diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index 5ffe4f8b92d4c7537acba276f7297dfc9534f406..5dbec576dc9a3f638de6c5a4466dc772c2b80d68 100644 (file)
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -168,9 +168,12 @@ public:
 
 
   // caps
-  void process_cap_update(int client, inodeno_t ino, __u64 cap_id, int caps, int wanted,
+  void process_cap_update(MDRequest *mdr, int client,
+                         inodeno_t ino, __u64 cap_id, int caps, int wanted,
                          int seq, int issue_seq, int mseq,
                          const nstring& dname);
+  void kick_cap_releases(MDRequest *mdr);
+
  protected:
   void adjust_cap_wanted(Capability *cap, int wanted, int issue_seq);
   void handle_client_caps(class MClientCaps *m);

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 156d4da43666c1dd938469653f4f70100ce3580f..cccd80926898e3fb879c07cb13e12f997bb4d204 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -6376,6 +6376,8 @@ void MDCache::request_cleanup(MDRequest *mdr)
        ++p) 
     (*p)->put_stickydirs();
 
+  mds->locker->kick_cap_releases(mdr);
+
   // drop cache pins
   mdr->drop_pins();
 

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 744d4657ea8a8d3a138a0de9611da4f6ada742d0..f74df29498e4cd6ec106cf2fa15f5d98bc777c6d 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -275,6 +275,9 @@ struct MDRequest : public Mutation {
   int snap_caps;
   bool did_early_reply;
 
+  // inos we did a embedded cap release on, and may need to eval if we haven't since reissued
+  map<vinodeno_t, ceph_seq_t> cap_releases;  
+
   // -- i am a slave request
   MMDSSlaveRequest *slave_request; // slave request (if one is pending; implies slave == true)
 

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 36e389a39a5e8e8c4126a893fd60bb02c1c8ebd5..2f093689b2aa8bcea2f0b7af7f24ff76312bdda4 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -690,9 +690,15 @@ void Server::early_reply(MDRequest *mdr, CInode *tracei, CDentry *tracedn)
           << " (" << strerror(-reply->get_result())
           << ") " << *req << dendl;
 
-  if (tracei || tracedn)
+  if (tracei || tracedn) {
+    if (tracei)
+      mdr->cap_releases.erase(tracei->vino());
+    if (tracedn)
+      mdr->cap_releases.erase(tracedn->get_dir()->get_inode()->vino());
+
     set_trace_dist(mdr->session, reply, tracei, tracedn, mdr->snapid,
                   mdr->client_request->get_dentry_wanted());
+  }
 
   messenger->send_message(reply, client_inst);
 
@@ -715,28 +721,8 @@ void Server::reply_request(MDRequest *mdr, MClientReply *reply, CInode *tracei,
   if (req->is_write() && mdr->session)
     mdr->session->add_completed_request(mdr->reqid.tid);
 
-  /*
-  if (tracei && !tracei->hack_accessed) {
-    tracei->hack_accessed = true;
-    mds->logger->inc("newt");
-    if (tracei->parent &&
-       tracei->parent->dir->hack_num_accessed >= 0) {
-      tracei->parent->dir->hack_num_accessed++;
-      if (tracei->parent->dir->hack_num_accessed == 1)
-       mds->logger->inc("dirt1");
-      if (tracei->parent->dir->hack_num_accessed == 2)
-       mds->logger->inc("dirt2");
-      if (tracei->parent->dir->hack_num_accessed == 3)
-       mds->logger->inc("dirt3");
-      if (tracei->parent->dir->hack_num_accessed == 4)
-       mds->logger->inc("dirt4");
-      if (tracei->parent->dir->hack_num_accessed == 5)
-       mds->logger->inc("dirt5");
-    }
-  }
-  */
-
-  reply->set_mdsmap_epoch(mds->mdsmap->get_epoch());
+  // give any preallocated inos to the session
+  apply_allocated_inos(mdr);
 
   // get tracei/tracedn from mdr?
   snapid_t snapid = mdr->snapid;
@@ -745,17 +731,21 @@ void Server::reply_request(MDRequest *mdr, MClientReply *reply, CInode *tracei,
   if (!tracedn)
     tracedn = mdr->tracedn;
 
-  // give any preallocated inos to the session
-  apply_allocated_inos(mdr);
-
   bool is_replay = mdr->client_request->is_replay();
-
-  // clean up request, drop locks, etc.
-  // do this before replying, so that we can issue leases
-  Session *session = mdr->session;
   bool did_early_reply = mdr->did_early_reply;
+  Session *session = mdr->session;
   entity_inst_t client_inst = req->get_orig_source_inst();
   int dentry_wanted = req->get_dentry_wanted();
+
+  if (!did_early_reply && !is_replay) {
+    if (tracei)
+      mdr->cap_releases.erase(tracei->vino());
+    if (tracedn)
+      mdr->cap_releases.erase(tracedn->get_dir()->get_inode()->vino());
+  }
+
+  // clean up request, drop locks, etc.
+  // do this before replying, so that we can issue leases
   mdcache->request_finish(mdr);
   mdr = 0;
 
@@ -775,6 +765,8 @@ void Server::reply_request(MDRequest *mdr, MClientReply *reply, CInode *tracei,
        set_trace_dist(session, reply, tracei, tracedn, snapid, dentry_wanted);
       }
     }
+
+    reply->set_mdsmap_epoch(mds->mdsmap->get_epoch());
     messenger->send_message(reply, client_inst);
   }
   
@@ -935,6 +927,13 @@ void Server::handle_client_request(MClientRequest *req)
     session->trim_completed_requests(req->get_oldest_client_tid());
   }
 
+  // register + dispatch
+  MDRequest *mdr = mdcache->request_start(req);
+  if (!mdr) 
+    return;
+  mdr->session = session;
+  session->requests.push_back(&mdr->session_request_item);
+
   // process embedded cap releases?
   //  (only if NOT replay!)
   if (req->get_source().is_client() &&
@@ -943,20 +942,14 @@ void Server::handle_client_request(MClientRequest *req)
     for (vector<MClientRequest::Release>::iterator p = req->releases.begin();
         p != req->releases.end();
         p++)
-      mds->locker->process_cap_update(client, inodeno_t((__u64)p->item.ino), p->item.cap_id,
+      mds->locker->process_cap_update(mdr, client,
+                                     inodeno_t((__u64)p->item.ino), p->item.cap_id,
                                      p->item.caps, p->item.wanted,
                                      p->item.seq, 
                                      p->item.issue_seq, 
                                      p->item.mseq, p->dname);
   }
 
-  // register + dispatch
-  MDRequest *mdr = mdcache->request_start(req);
-  if (!mdr) 
-    return;
-  mdr->session = session;
-  session->requests.push_back(&mdr->session_request_item);
-
   dispatch_client_request(mdr);
   return;
 }