mds: fix occasional dir rstat inconsistency between multi-MDSes

Currently if file inode and its parent are processed on auth MDS,
its parent's rstat will be updated correctly, but won't be passed
to non-auth MDS. So if a newly-mounted ceph-fuse client connects
to non-auth MDS, this parent's rstat might be old.

The fix is to forward getattr request with CEPH_STAT_RSTAT mask
to auth MDS for latest rstat and client will always update its rstat
if reply is from auth MDS.

Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
(cherry picked from commit 7a2e42852f6711f0c470de07063bac0f47a72a30)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 1fd435839490a2d240a7a4d207d292aa45871559..b70086398b0275a78da994096e78b057ae53879c 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -902,8 +902,10 @@ Inode * Client::add_update_inode(InodeStat *st, utime_t from,
     add_update_cap(in, session, st->cap.cap_id, st->cap.caps, st->cap.seq,
                   st->cap.mseq, inodeno_t(st->cap.realm), st->cap.flags,
                   request_perms);
-    if (in->auth_cap && in->auth_cap->session == session)
+    if (in->auth_cap && in->auth_cap->session == session) {
       in->max_size = st->max_size;
+      in->rstat = st->rstat;
+    }
   } else
     in->snap_caps |= st->cap.caps;
 
@@ -10852,7 +10854,11 @@ int Client::_getxattr(Inode *in, const char *name, void *value, size_t size,
 
     // Do a force getattr to get the latest quota before returning
     // a value to userspace.
-    r = _getattr(in, 0, perms, true);
+    int flags = 0;
+    if (vxattr->flags & VXATTR_RSTAT) {
+      flags |= CEPH_STAT_RSTAT;
+    }
+    r = _getattr(in, flags, perms, true);
     if (r != 0) {
       // Error from getattr!
       return r;
@@ -11369,6 +11375,16 @@ size_t Client::_vxattrcb_dir_rctime(Inode *in, char *val, size_t size)
   readonly: true,                                              \
   hidden: false,                                               \
   exists_cb: NULL,                                             \
+  flags: 0,                                                     \
+}
+#define XATTR_NAME_CEPH2(_type, _name, _flags)                 \
+{                                                              \
+  name: CEPH_XATTR_NAME(_type, _name),                         \
+  getxattr_cb: &Client::_vxattrcb_ ## _type ## _ ## _name,     \
+  readonly: true,                                              \
+  hidden: false,                                               \
+  exists_cb: NULL,                                             \
+  flags: _flags,                                               \
 }
 #define XATTR_LAYOUT_FIELD(_type, _name, _field)               \
 {                                                              \
@@ -11377,6 +11393,7 @@ size_t Client::_vxattrcb_dir_rctime(Inode *in, char *val, size_t size)
   readonly: false,                                             \
   hidden: true,                                                        \
   exists_cb: &Client::_vxattrcb_layout_exists,                 \
+  flags: 0,                                                     \
 }
 #define XATTR_QUOTA_FIELD(_type, _name)                                \
 {                                                              \
@@ -11385,6 +11402,7 @@ size_t Client::_vxattrcb_dir_rctime(Inode *in, char *val, size_t size)
   readonly: false,                                             \
   hidden: true,                                                        \
   exists_cb: &Client::_vxattrcb_quota_exists,                  \
+  flags: 0,                                                     \
 }
 
 const Client::VXattr Client::_dir_vxattrs[] = {
@@ -11394,6 +11412,7 @@ const Client::VXattr Client::_dir_vxattrs[] = {
     readonly: false,
     hidden: true,
     exists_cb: &Client::_vxattrcb_layout_exists,
+    flags: 0,
   },
   XATTR_LAYOUT_FIELD(dir, layout, stripe_unit),
   XATTR_LAYOUT_FIELD(dir, layout, stripe_count),
@@ -11403,17 +11422,18 @@ const Client::VXattr Client::_dir_vxattrs[] = {
   XATTR_NAME_CEPH(dir, entries),
   XATTR_NAME_CEPH(dir, files),
   XATTR_NAME_CEPH(dir, subdirs),
-  XATTR_NAME_CEPH(dir, rentries),
-  XATTR_NAME_CEPH(dir, rfiles),
-  XATTR_NAME_CEPH(dir, rsubdirs),
-  XATTR_NAME_CEPH(dir, rbytes),
-  XATTR_NAME_CEPH(dir, rctime),
+  XATTR_NAME_CEPH2(dir, rentries, VXATTR_RSTAT),
+  XATTR_NAME_CEPH2(dir, rfiles, VXATTR_RSTAT),
+  XATTR_NAME_CEPH2(dir, rsubdirs, VXATTR_RSTAT),
+  XATTR_NAME_CEPH2(dir, rbytes, VXATTR_RSTAT),
+  XATTR_NAME_CEPH2(dir, rctime, VXATTR_RSTAT),
   {
     name: "ceph.quota",
     getxattr_cb: &Client::_vxattrcb_quota,
     readonly: false,
     hidden: true,
     exists_cb: &Client::_vxattrcb_quota_exists,
+    flags: 0,
   },
   XATTR_QUOTA_FIELD(quota, max_bytes),
   XATTR_QUOTA_FIELD(quota, max_files),
@@ -11427,6 +11447,7 @@ const Client::VXattr Client::_file_vxattrs[] = {
     readonly: false,
     hidden: true,
     exists_cb: &Client::_vxattrcb_layout_exists,
+    flags: 0,
   },
   XATTR_LAYOUT_FIELD(file, layout, stripe_unit),
   XATTR_LAYOUT_FIELD(file, layout, stripe_count),

diff --git a/src/client/Client.h b/src/client/Client.h
index 0d7cd787bad13ec4ee5c6256d1ec6c271441d0ce..e838a9b1d78dec76e694b73fac209fa771bc6bf7 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -891,8 +891,12 @@ private:
          size_t (Client::*getxattr_cb)(Inode *in, char *val, size_t size);
          bool readonly, hidden;
          bool (Client::*exists_cb)(Inode *in);
+         int flags;
   };
 
+/* Flags for VXattr */
+#define VXATTR_RSTAT 0x1
+
   bool _vxattrcb_quota_exists(Inode *in);
   size_t _vxattrcb_quota(Inode *in, char *val, size_t size);
   size_t _vxattrcb_quota_max_bytes(Inode *in, char *val, size_t size);

diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index 4ddfda5be7b74eab727eadc513a8ca23f8bffa2d..74af918c6170fd9bf5cd68edb39c9653c60a00a3 100644 (file)
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -733,6 +733,7 @@ int ceph_flags_to_mode(int flags);
                                 CEPH_CAP_XATTR_SHARED)
 #define CEPH_STAT_CAP_INLINE_DATA (CEPH_CAP_FILE_SHARED | \
                                   CEPH_CAP_FILE_RD)
+#define CEPH_STAT_RSTAT        (CEPH_CAP_GWREXTEND << 16)  /* for requesting rstat */
 
 #define CEPH_CAP_ANY_SHARED (CEPH_CAP_AUTH_SHARED |                    \
                              CEPH_CAP_LINK_SHARED |                    \

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 986f6e6b876c474755c832f6e2fb0aaacc8a9c0d..d0f171810aa222ae2c97f5404e008ed52faf7d80 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -3004,7 +3004,13 @@ void Server::handle_client_getattr(MDRequestRef& mdr, bool is_lookup)
     return;
   }
 
-  CInode *ref = rdlock_path_pin_ref(mdr, 0, rdlocks, false, false, NULL, !is_lookup);
+  bool want_auth = false;
+  int mask = req->head.args.getattr.mask;
+  if (mask & CEPH_STAT_RSTAT)
+    want_auth = true; // set want_auth for CEPH_STAT_RSTAT mask
+
+  CInode *ref = rdlock_path_pin_ref(mdr, 0, rdlocks, want_auth, false, NULL, 
+                                   !is_lookup);
   if (!ref) return;
 
   /*
@@ -3022,7 +3028,6 @@ void Server::handle_client_getattr(MDRequestRef& mdr, bool is_lookup)
              mdr->snapid <= cap->client_follows))
     issued = cap->issued();
 
-  int mask = req->head.args.getattr.mask;
   if ((mask & CEPH_CAP_LINK_SHARED) && !(issued & CEPH_CAP_LINK_EXCL))
     rdlocks.insert(&ref->linklock);
   if ((mask & CEPH_CAP_AUTH_SHARED) && !(issued & CEPH_CAP_AUTH_EXCL))