From: Gui Hecheng Date: Fri, 26 May 2017 09:45:29 +0000 (+0800) Subject: rgw_file: new fsid for fs instance X-Git-Tag: v13.0.0~145^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=178b547e83a2c0eeb024bbe4ca2ff8d8e7d50572;p=ceph.git rgw_file: new fsid for fs instance We use an in-memory fs_inst_counter for fsid currently, but it inevitably cause nfs automount problem across a nfs-ganesha crash. So here we relate the fsid to the s3 uid with a hash. With this, fsid remains consistent across all fs instances and across crashes. We should be able to upgrade from old fsid to new fsid with update_fh(). Signed-off-by: Gui Hecheng --- diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc index 83df3ddbf01b..60aaa2dc6e10 100644 --- a/src/rgw/rgw_file.cc +++ b/src/rgw/rgw_file.cc @@ -92,10 +92,10 @@ namespace rgw { auto ux_key = req.get_attr(RGW_ATTR_UNIX_KEY1); auto ux_attrs = req.get_attr(RGW_ATTR_UNIX1); if (ux_key && ux_attrs) { - bool old_key = rgw_fh->decode_attrs(ux_key, ux_attrs); - if (old_key) { - update_fhk(rgw_fh); - } + DecodeAttrsResult dar = rgw_fh->decode_attrs(ux_key, ux_attrs); + if (get<0>(dar) || get<1>(dar)) { + update_fh(rgw_fh); + } } if (! (flags & RGWFileHandle::FLAG_LOCKED)) { rgw_fh->mtx.unlock(); @@ -147,10 +147,10 @@ namespace rgw { auto ux_key = req.get_attr(RGW_ATTR_UNIX_KEY1); auto ux_attrs = req.get_attr(RGW_ATTR_UNIX1); if (ux_key && ux_attrs) { - bool old_key = rgw_fh->decode_attrs(ux_key, ux_attrs); - if (old_key) { - update_fhk(rgw_fh); - } + DecodeAttrsResult dar = rgw_fh->decode_attrs(ux_key, ux_attrs); + if (get<0>(dar) || get<1>(dar)) { + update_fh(rgw_fh); + } } } goto done; @@ -181,10 +181,10 @@ namespace rgw { auto ux_key = req.get_attr(RGW_ATTR_UNIX_KEY1); auto ux_attrs = req.get_attr(RGW_ATTR_UNIX1); if (ux_key && ux_attrs) { - bool old_key = rgw_fh->decode_attrs(ux_key, ux_attrs); - if (old_key) { - update_fhk(rgw_fh); - } + DecodeAttrsResult dar = rgw_fh->decode_attrs(ux_key, ux_attrs); + if (get<0>(dar) || get<1>(dar)) { + update_fh(rgw_fh); + } } } goto done; @@ -744,7 +744,7 @@ namespace rgw { } /* RGWLibFS::setattr */ /* called under rgw_fh->mtx held */ - void RGWLibFS::update_fhk(RGWFileHandle *rgw_fh) + void RGWLibFS::update_fh(RGWFileHandle *rgw_fh) { int rc, rc2; string obj_name{rgw_fh->relative_object_name()}; @@ -757,15 +757,15 @@ namespace rgw { lsubdout(get_context(), rgw, 17) << __func__ - << " update old versioned fhk : " << obj_name + << " update old versioned fh : " << obj_name << dendl; RGWSetAttrsRequest req(cct, get_user(), rgw_fh->bucket_name(), obj_name); rgw_fh->encode_attrs(ux_key, ux_attrs); - /* update ux_key only */ req.emplace_attr(RGW_ATTR_UNIX_KEY1, std::move(ux_key)); + req.emplace_attr(RGW_ATTR_UNIX1, std::move(ux_attrs)); rc = rgwlib.get_fe()->execute_req(&req); rc2 = req.get_ret(); @@ -773,10 +773,10 @@ namespace rgw { if ((rc != 0) || (rc2 != 0)) { lsubdout(get_context(), rgw, 17) << __func__ - << " update fhk failed : " << obj_name + << " update fh failed : " << obj_name << dendl; } - } /* RGWLibFS::update_fhk */ + } /* RGWLibFS::update_fh */ void RGWLibFS::close() { @@ -786,7 +786,7 @@ namespace rgw { { RGWLibFS* fs; public: - ObjUnref(RGWLibFS* fs) : fs(fs) {} + ObjUnref(RGWLibFS* _fs) : fs(_fs) {} void operator()(RGWFileHandle* fh) const { lsubdout(fs->get_context(), rgw, 5) << __func__ @@ -976,23 +976,26 @@ namespace rgw { rgw::encode(*this, ux_attrs1); } /* RGWFileHandle::encode_attrs */ - bool RGWFileHandle::decode_attrs(const ceph::buffer::list* ux_key1, - const ceph::buffer::list* ux_attrs1) + DecodeAttrsResult RGWFileHandle::decode_attrs(const ceph::buffer::list* ux_key1, + const ceph::buffer::list* ux_attrs1) { - bool old_key = false; + DecodeAttrsResult dar { false, false }; fh_key fhk; auto bl_iter_key1 = const_cast(ux_key1)->begin(); rgw::decode(fhk, bl_iter_key1); if (fhk.version >= 2) { assert(this->fh.fh_hk == fhk.fh_hk); } else { - old_key = true; + get<0>(dar) = true; } auto bl_iter_unix1 = const_cast(ux_attrs1)->begin(); rgw::decode(*this, bl_iter_unix1); + if (this->state.version < 2) { + get<1>(dar) = true; + } - return old_key; + return dar; } /* RGWFileHandle::decode_attrs */ bool RGWFileHandle::reclaim() { @@ -1566,8 +1569,8 @@ int rgw_statfs(struct rgw_fs *rgw_fs, vfs_st->f_bavail = UINT64_MAX; vfs_st->f_files = 1024; /* object count, do we have an est? */ vfs_st->f_ffree = UINT64_MAX; - vfs_st->f_fsid[0] = fs->get_inst(); - vfs_st->f_fsid[1] = fs->get_inst(); + vfs_st->f_fsid[0] = fs->get_fsid(); + vfs_st->f_fsid[1] = fs->get_fsid(); vfs_st->f_flag = 0; vfs_st->f_namemax = 4096; return 0; diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h index 34f3f0bcec86..8bb702cb33fe 100644 --- a/src/rgw/rgw_file.h +++ b/src/rgw/rgw_file.h @@ -173,6 +173,8 @@ namespace rgw { using boost::variant; using boost::container::flat_map; + typedef std::tuple DecodeAttrsResult; + class RGWFileHandle : public cohort::lru::Object { struct rgw_file_handle fh; @@ -204,8 +206,9 @@ namespace rgw { struct timespec ctime; struct timespec mtime; struct timespec atime; + uint32_t version; State() : dev(0), size(0), nlink(1), owner_uid(0), owner_gid(0), - ctime{0,0}, mtime{0,0}, atime{0,0} {} + ctime{0,0}, mtime{0,0}, atime{0,0}, version(0) {} } state; struct file { @@ -257,7 +260,7 @@ namespace rgw { friend class RGWLibFS; private: - RGWFileHandle(RGWLibFS* _fs, uint32_t fs_inst) + RGWFileHandle(RGWLibFS* _fs) : fs(_fs), bucket(nullptr), parent(nullptr), variant_type{directory()}, depth(0), flags(FLAG_ROOT) { @@ -265,12 +268,15 @@ namespace rgw { fh.fh_type = RGW_FS_TYPE_DIRECTORY; variant_type = directory(); /* stat */ - state.dev = fs_inst; state.unix_mode = RGW_RWXMODE|S_IFDIR; /* pointer to self */ fh.fh_private = this; } + uint64_t init_fsid(std::string& uid) { + return XXH64(uid.c_str(), uid.length(), fh_key::seed); + } + void init_rootfs(std::string& fsid, const std::string& object_name) { /* fh_key */ fh.fh_hk.bucket = XXH64(fsid.c_str(), fsid.length(), fh_key::seed); @@ -278,12 +284,14 @@ namespace rgw { fh_key::seed); fhk = fh.fh_hk; name = object_name; + + state.dev = init_fsid(fsid); } public: - RGWFileHandle(RGWLibFS* fs, uint32_t fs_inst, RGWFileHandle* _parent, + RGWFileHandle(RGWLibFS* _fs, RGWFileHandle* _parent, const fh_key& _fhk, std::string& _name, uint32_t _flags) - : fs(fs), bucket(nullptr), parent(_parent), name(std::move(_name)), + : fs(_fs), bucket(nullptr), parent(_parent), name(std::move(_name)), fhk(_fhk), flags(_flags) { if (parent->is_root()) { @@ -307,8 +315,8 @@ namespace rgw { /* save constant fhk */ fh.fh_hk = fhk.fh_hk; /* XXX redundant in fh_hk */ - /* stat */ - state.dev = fs_inst; + /* inherits parent's fsid */ + state.dev = parent->state.dev; switch (fh.fh_type) { case RGW_FS_TYPE_DIRECTORY: @@ -594,7 +602,7 @@ namespace rgw { } void encode(buffer::list& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); ::encode(uint32_t(fh.fh_type), bl); ::encode(state.dev, bl); ::encode(state.size, bl); @@ -605,11 +613,12 @@ namespace rgw { for (const auto& t : { state.ctime, state.mtime, state.atime }) { ::encode(real_clock::from_timespec(t), bl); } + ::encode((uint32_t)2, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); + DECODE_START(2, bl); uint32_t fh_type; ::decode(fh_type, bl); assert(fh.fh_type == fh_type); @@ -624,14 +633,17 @@ namespace rgw { ::decode(enc_time, bl); *t = real_clock::to_timespec(enc_time); } + if (struct_v >= 2) { + ::decode(state.version, bl); + } DECODE_FINISH(bl); } void encode_attrs(ceph::buffer::list& ux_key1, ceph::buffer::list& ux_attrs1); - bool decode_attrs(const ceph::buffer::list* ux_key1, - const ceph::buffer::list* ux_attrs1); + DecodeAttrsResult decode_attrs(const ceph::buffer::list* ux_key1, + const ceph::buffer::list* ux_attrs1); void invalidate(); @@ -694,7 +706,6 @@ namespace rgw { { public: RGWLibFS* fs; - uint32_t fs_inst; RGWFileHandle* parent; const fh_key& fhk; std::string& name; @@ -702,20 +713,20 @@ namespace rgw { Factory() = delete; - Factory(RGWLibFS* fs, uint32_t fs_inst, RGWFileHandle* parent, - const fh_key& fhk, std::string& name, uint32_t flags) - : fs(fs), fs_inst(fs_inst), parent(parent), fhk(fhk), name(name), - flags(flags) {} + Factory(RGWLibFS* _fs, RGWFileHandle* _parent, + const fh_key& _fhk, std::string& _name, uint32_t _flags) + : fs(_fs), parent(_parent), fhk(_fhk), name(_name), + flags(_flags) {} void recycle (cohort::lru::Object* o) override { /* re-use an existing object */ o->~Object(); // call lru::Object virtual dtor // placement new! - new (o) RGWFileHandle(fs, fs_inst, parent, fhk, name, flags); + new (o) RGWFileHandle(fs, parent, fhk, name, flags); } cohort::lru::Object* alloc() override { - return new RGWFileHandle(fs, fs_inst, parent, fhk, name, flags); + return new RGWFileHandle(fs, parent, fhk, name, flags); } }; /* Factory */ @@ -768,7 +779,6 @@ namespace rgw { static std::atomic fs_inst_counter; static uint32_t write_completion_interval_s; - std::string fsid; using lock_guard = std::lock_guard; using unique_lock = std::unique_lock; @@ -838,7 +848,7 @@ namespace rgw { RGWLibFS(CephContext* _cct, const char *_uid, const char *_user_id, const char* _key) - : cct(_cct), root_fh(this, new_inst()), invalidate_cb(nullptr), + : cct(_cct), root_fh(this), invalidate_cb(nullptr), invalidate_arg(nullptr), shutdown(false), refcnt(1), fh_cache(cct->_conf->rgw_nfs_fhcache_partitions, cct->_conf->rgw_nfs_fhcache_size), @@ -846,17 +856,15 @@ namespace rgw { cct->_conf->rgw_nfs_lru_lane_hiwat), uid(_uid), key(_user_id, _key) { - /* no bucket may be named rgw_fs_inst-(.*) */ - fsid = RGWFileHandle::root_name + "rgw_fs_inst-" + - std::to_string(get_inst()); - - root_fh.init_rootfs(fsid /* bucket */, RGWFileHandle::root_name); + root_fh.init_rootfs(uid, RGWFileHandle::root_name); /* pointer to self */ fs.fs_private = this; /* expose public root fh */ fs.root_fh = root_fh.get_fh(); + + new_inst(); } friend void intrusive_ptr_add_ref(const RGWLibFS* fs) { @@ -1031,7 +1039,7 @@ namespace rgw { fh->mtx.unlock(); /* ! LOCKED */ } else { /* make or re-use handle */ - RGWFileHandle::Factory prototype(this, get_inst(), parent, fhk, + RGWFileHandle::Factory prototype(this, parent, fhk, obj_name, CREATE_FLAGS(flags)); fh = static_cast( fh_lru.insert(&prototype, @@ -1083,8 +1091,7 @@ namespace rgw { int setattr(RGWFileHandle* rgw_fh, struct stat* st, uint32_t mask, uint32_t flags); - void update_fhk(RGWFileHandle *rgw_fh); - + void update_fh(RGWFileHandle *rgw_fh); LookupFHResult stat_bucket(RGWFileHandle* parent, const char *path, RGWLibFS::BucketStats& bs, @@ -1166,7 +1173,7 @@ namespace rgw { struct rgw_fs* get_fs() { return &fs; } - uint32_t get_inst() { return root_fh.state.dev; } + uint64_t get_fsid() { return root_fh.state.dev; } RGWUserInfo* get_user() { return &user; }