From 7a2e42852f6711f0c470de07063bac0f47a72a30 Mon Sep 17 00:00:00 2001 From: Zhi Zhang Date: Tue, 17 Apr 2018 11:14:03 +0800 Subject: [PATCH] mds: fix occasional dir rstat inconsistency between multi-MDSes Currently if file inode and its parent are processed on auth MDS, its parent's rstat will be updated correctly, but won't be passed to non-auth MDS. So if a newly-mounted ceph-fuse client connects to non-auth MDS, this parent's rstat might be old. The fix is to forward getattr request with CEPH_STAT_RSTAT mask to auth MDS for latest rstat and client will always update its rstat if reply is from auth MDS. Signed-off-by: Zhi Zhang --- src/client/Client.cc | 35 ++++++++++++++++++++++++++++------- src/client/Client.h | 4 ++++ src/include/ceph_fs.h | 1 + src/mds/Server.cc | 9 +++++++-- 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index 281f9d33548..e4baa570a95 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -903,8 +903,10 @@ Inode * Client::add_update_inode(InodeStat *st, utime_t from, add_update_cap(in, session, st->cap.cap_id, st->cap.caps, st->cap.seq, st->cap.mseq, inodeno_t(st->cap.realm), st->cap.flags, request_perms); - if (in->auth_cap && in->auth_cap->session == session) + if (in->auth_cap && in->auth_cap->session == session) { in->max_size = st->max_size; + in->rstat = st->rstat; + } } else in->snap_caps |= st->cap.caps; @@ -10882,7 +10884,11 @@ int Client::_getxattr(Inode *in, const char *name, void *value, size_t size, // Do a force getattr to get the latest quota before returning // a value to userspace. - r = _getattr(in, 0, perms, true); + int flags = 0; + if (vxattr->flags & VXATTR_RSTAT) { + flags |= CEPH_STAT_RSTAT; + } + r = _getattr(in, flags, perms, true); if (r != 0) { // Error from getattr! return r; @@ -11406,6 +11412,16 @@ size_t Client::_vxattrcb_dir_rctime(Inode *in, char *val, size_t size) readonly: true, \ hidden: false, \ exists_cb: NULL, \ + flags: 0, \ +} +#define XATTR_NAME_CEPH2(_type, _name, _flags) \ +{ \ + name: CEPH_XATTR_NAME(_type, _name), \ + getxattr_cb: &Client::_vxattrcb_ ## _type ## _ ## _name, \ + readonly: true, \ + hidden: false, \ + exists_cb: NULL, \ + flags: _flags, \ } #define XATTR_LAYOUT_FIELD(_type, _name, _field) \ { \ @@ -11414,6 +11430,7 @@ size_t Client::_vxattrcb_dir_rctime(Inode *in, char *val, size_t size) readonly: false, \ hidden: true, \ exists_cb: &Client::_vxattrcb_layout_exists, \ + flags: 0, \ } #define XATTR_QUOTA_FIELD(_type, _name) \ { \ @@ -11422,6 +11439,7 @@ size_t Client::_vxattrcb_dir_rctime(Inode *in, char *val, size_t size) readonly: false, \ hidden: true, \ exists_cb: &Client::_vxattrcb_quota_exists, \ + flags: 0, \ } const Client::VXattr Client::_dir_vxattrs[] = { @@ -11431,6 +11449,7 @@ const Client::VXattr Client::_dir_vxattrs[] = { readonly: false, hidden: true, exists_cb: &Client::_vxattrcb_layout_exists, + flags: 0, }, XATTR_LAYOUT_FIELD(dir, layout, stripe_unit), XATTR_LAYOUT_FIELD(dir, layout, stripe_count), @@ -11440,17 +11459,18 @@ const Client::VXattr Client::_dir_vxattrs[] = { XATTR_NAME_CEPH(dir, entries), XATTR_NAME_CEPH(dir, files), XATTR_NAME_CEPH(dir, subdirs), - XATTR_NAME_CEPH(dir, rentries), - XATTR_NAME_CEPH(dir, rfiles), - XATTR_NAME_CEPH(dir, rsubdirs), - XATTR_NAME_CEPH(dir, rbytes), - XATTR_NAME_CEPH(dir, rctime), + XATTR_NAME_CEPH2(dir, rentries, VXATTR_RSTAT), + XATTR_NAME_CEPH2(dir, rfiles, VXATTR_RSTAT), + XATTR_NAME_CEPH2(dir, rsubdirs, VXATTR_RSTAT), + XATTR_NAME_CEPH2(dir, rbytes, VXATTR_RSTAT), + XATTR_NAME_CEPH2(dir, rctime, VXATTR_RSTAT), { name: "ceph.quota", getxattr_cb: &Client::_vxattrcb_quota, readonly: false, hidden: true, exists_cb: &Client::_vxattrcb_quota_exists, + flags: 0, }, XATTR_QUOTA_FIELD(quota, max_bytes), XATTR_QUOTA_FIELD(quota, max_files), @@ -11464,6 +11484,7 @@ const Client::VXattr Client::_file_vxattrs[] = { readonly: false, hidden: true, exists_cb: &Client::_vxattrcb_layout_exists, + flags: 0, }, XATTR_LAYOUT_FIELD(file, layout, stripe_unit), XATTR_LAYOUT_FIELD(file, layout, stripe_count), diff --git a/src/client/Client.h b/src/client/Client.h index e578add5d1f..eb48c68c1e3 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -891,8 +891,12 @@ private: size_t (Client::*getxattr_cb)(Inode *in, char *val, size_t size); bool readonly, hidden; bool (Client::*exists_cb)(Inode *in); + int flags; }; +/* Flags for VXattr */ +#define VXATTR_RSTAT 0x1 + bool _vxattrcb_quota_exists(Inode *in); size_t _vxattrcb_quota(Inode *in, char *val, size_t size); size_t _vxattrcb_quota_max_bytes(Inode *in, char *val, size_t size); diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h index 6f755fbdee8..a4cd388e70f 100644 --- a/src/include/ceph_fs.h +++ b/src/include/ceph_fs.h @@ -739,6 +739,7 @@ int ceph_flags_to_mode(int flags); CEPH_CAP_XATTR_SHARED) #define CEPH_STAT_CAP_INLINE_DATA (CEPH_CAP_FILE_SHARED | \ CEPH_CAP_FILE_RD) +#define CEPH_STAT_RSTAT (CEPH_CAP_GWREXTEND << 16) /* for requesting rstat */ #define CEPH_CAP_ANY_SHARED (CEPH_CAP_AUTH_SHARED | \ CEPH_CAP_LINK_SHARED | \ diff --git a/src/mds/Server.cc b/src/mds/Server.cc index d2c29d8ebd0..c0ae38a0a8b 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -3016,7 +3016,13 @@ void Server::handle_client_getattr(MDRequestRef& mdr, bool is_lookup) return; } - CInode *ref = rdlock_path_pin_ref(mdr, 0, rdlocks, false, false, NULL, !is_lookup); + bool want_auth = false; + int mask = req->head.args.getattr.mask; + if (mask & CEPH_STAT_RSTAT) + want_auth = true; // set want_auth for CEPH_STAT_RSTAT mask + + CInode *ref = rdlock_path_pin_ref(mdr, 0, rdlocks, want_auth, false, NULL, + !is_lookup); if (!ref) return; /* @@ -3034,7 +3040,6 @@ void Server::handle_client_getattr(MDRequestRef& mdr, bool is_lookup) mdr->snapid <= cap->client_follows)) issued = cap->issued(); - int mask = req->head.args.getattr.mask; if ((mask & CEPH_CAP_LINK_SHARED) && !(issued & CEPH_CAP_LINK_EXCL)) rdlocks.insert(&ref->linklock); if ((mask & CEPH_CAP_AUTH_SHARED) && !(issued & CEPH_CAP_AUTH_EXCL)) -- 2.39.5