mds: fix corner case of pushing inline data

Following sequence of events can happen.
 - Client releases an inode, queues cap release message.
 - A 'lookup' reply brings the same inode back, but the reply doesn't
   contain inline data because MDS didn't receive the cap release
   message and thought client already has up-to-data inline data.

The fix is trigger a getattr if client finds inline_version is zero.
The getattr mask is set to CEPH_STAT_CAP_INLINE_DATA, so that MDS knows
client does not have inline data.

Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 554732745b7ae173ad7ec5eafb59607075e1fe75..8b5a036e5138f280063d687882772802ba560484 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -679,6 +679,9 @@ Inode * Client::add_update_inode(InodeStat *st, utime_t from,
                           st->time_warp_seq, st->ctime, st->mtime, st->atime,
                           st->inline_version, st->inline_data,
                           issued);
+  } else if (st->inline_version > in->inline_version) {
+    in->inline_data = st->inline_data;
+    in->inline_version = st->inline_version;
   }
 
   // move me if/when version reflects fragtree changes.
@@ -5887,6 +5890,13 @@ int Client::_read(Fh *f, int64_t offset, uint64_t size, bufferlist *bl)
 
   //bool lazy = f->mode == CEPH_FILE_MODE_LAZY;
 
+  if (in->inline_version == 0) {
+    int r = _getattr(in, CEPH_STAT_CAP_INLINE_DATA);
+    if (r < 0)
+      return r;
+    assert(in->inline_version > 0);
+  }
+
   int have;
   int r = get_caps(in, CEPH_CAP_FILE_RD, CEPH_CAP_FILE_CACHE, &have, -1);
   if (r < 0)
@@ -6245,6 +6255,13 @@ int Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf)
   // time it.
   utime_t start = ceph_clock_now(cct);
 
+  if (in->inline_version == 0) {
+    int r = _getattr(in, CEPH_STAT_CAP_INLINE_DATA);
+    if (r < 0)
+      return r;
+    assert(in->inline_version > 0);
+  }
+
   // copy into fresh buffer (since our write may be resub, async)
   bufferptr bp;
   if (size > 0) bp = buffer::copy(buf, size);

diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index 12bdd967c50adc0932044987f49c7bd39a48b5fb..10b52a5033abe1792e069e1e03331e74e162ede8 100644 (file)
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -599,6 +599,8 @@ int ceph_flags_to_mode(int flags);
                                 CEPH_CAP_LINK_SHARED | \
                                 CEPH_CAP_FILE_SHARED | \
                                 CEPH_CAP_XATTR_SHARED)
+#define CEPH_STAT_CAP_INLINE_DATA (CEPH_CAP_FILE_SHARED | \
+                                  CEPH_CAP_FILE_RD)
 
 #define CEPH_CAP_ANY_SHARED (CEPH_CAP_AUTH_SHARED |                    \
                              CEPH_CAP_LINK_SHARED |                    \

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 1ce2ee80b7c4c210b83521529c496a39e329240d..7feffa1d0d0a9329b1db254cef80d0fd73c7649e 100644 (file)
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -2880,11 +2880,13 @@ int CInode::encode_inodestat(bufferlist& bl, Session *session,
   // inline data
   version_t inline_version = 0;
   bufferlist inline_data;
-  if (!cap || (cap->client_inline_version < i->inline_version)) {
+  if (i->inline_version == CEPH_INLINE_NONE) {
+    inline_version = CEPH_INLINE_NONE;
+  } else if ((!cap && !no_caps) ||
+            (cap && cap->client_inline_version < i->inline_version) ||
+            (getattr_caps & CEPH_CAP_FILE_RD)) { // client requests inline data
     inline_version = i->inline_version;
     inline_data = i->inline_data;
-    if (cap)
-      cap->client_inline_version = i->inline_version;
   }
 
   // nest (do same as file... :/)
@@ -2995,6 +2997,17 @@ int CInode::encode_inodestat(bufferlist& bl, Session *session,
           << " xattrv " << e.xattr_version << " len " << xbl.length()
           << dendl;
 
+  if (inline_data.length() && cap) {
+    if ((cap->pending() | getattr_caps) & CEPH_CAP_FILE_SHARED) {
+      dout(10) << "including inline version " << inline_version << dendl;
+      cap->client_inline_version = inline_version;
+    } else {
+      dout(10) << "dropping inline version " << inline_version << dendl;
+      inline_version = 0;
+      inline_data.clear();
+    }
+  }
+
   // include those xattrs?
   if (xbl.length() && cap) {
     if ((cap->pending() | getattr_caps) & CEPH_CAP_XATTR_SHARED) {

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index a1327662d9b2212bc6e6596ee841dde9d8b6164e..8a4321e00fc2350d1455a1b647da8881350ef15c 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -2350,7 +2350,7 @@ void Server::handle_client_getattr(MDRequest *mdr, bool is_lookup)
     return;
 
   // note which caps are requested, so we return at least a snapshot
-  // value for them.  (currently this only matters for xattrs)
+  // value for them.  (currently this matters for xattrs and inline data)
   mdr->getattr_caps = mask;
 
   mds->balancer->hit_inode(ceph_clock_now(g_ceph_context), ref, META_POP_IRD,