From d997f657750faf920170843e62deacab70008d8b Mon Sep 17 00:00:00 2001 From: "Adam C. Emerson" Date: Wed, 20 Dec 2017 17:06:32 -0500 Subject: [PATCH] rgw: Plumb refresh logic into object cache Now when we force a refetch of bucket info it will actually go to the OSD rather than simply using the objects in the object cache. Fixes: http://tracker.ceph.com/issues/22517 Signed-off-by: Adam C. Emerson --- src/rgw/rgw_cache.h | 11 +++++++---- src/rgw/rgw_rados.cc | 32 +++++++++++++++++++++++--------- src/rgw/rgw_rados.h | 13 +++++++++---- src/rgw/rgw_tools.cc | 4 ++-- src/rgw/rgw_tools.h | 3 ++- 5 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/rgw/rgw_cache.h b/src/rgw/rgw_cache.h index faabc0cda2fd..880001647af8 100644 --- a/src/rgw/rgw_cache.h +++ b/src/rgw/rgw_cache.h @@ -243,7 +243,8 @@ public: RGWObjVersionTracker *objv_tracker, rgw_raw_obj& obj, bufferlist& bl, off_t ofs, off_t end, map *attrs, - rgw_cache_entry_info *cache_info) override; + rgw_cache_entry_info *cache_info, + boost::optional refresh_version = boost::none) override; int raw_obj_stat(rgw_raw_obj& obj, uint64_t *psize, real_time *pmtime, uint64_t *epoch, map *attrs, bufferlist *first_chunk, RGWObjVersionTracker *objv_tracker) override; @@ -288,7 +289,8 @@ int RGWCache::get_system_obj(RGWObjectCtx& obj_ctx, RGWRados::SystemObject::R RGWObjVersionTracker *objv_tracker, rgw_raw_obj& obj, bufferlist& obl, off_t ofs, off_t end, map *attrs, - rgw_cache_entry_info *cache_info) + rgw_cache_entry_info *cache_info, + boost::optional refresh_version) { rgw_pool pool; string oid; @@ -305,8 +307,9 @@ int RGWCache::get_system_obj(RGWObjectCtx& obj_ctx, RGWRados::SystemObject::R flags |= CACHE_FLAG_OBJV; if (attrs) flags |= CACHE_FLAG_XATTRS; - - if (cache.get(name, info, flags, cache_info) == 0) { + + if ((cache.get(name, info, flags, cache_info) == 0) && + (!refresh_version || !info.version.compare(&(*refresh_version)))) { if (info.status < 0) return info.status; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 4bca3138b425..1073b85a8355 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -10225,7 +10225,8 @@ int RGWRados::get_system_obj(RGWObjectCtx& obj_ctx, RGWRados::SystemObject::Read RGWObjVersionTracker *objv_tracker, rgw_raw_obj& obj, bufferlist& bl, off_t ofs, off_t end, map *attrs, - rgw_cache_entry_info *cache_info) + rgw_cache_entry_info *cache_info, + boost::optional) { uint64_t len; ObjectReadOperation op; @@ -10272,12 +10273,16 @@ int RGWRados::get_system_obj(RGWObjectCtx& obj_ctx, RGWRados::SystemObject::Read return bl.length(); } -int RGWRados::SystemObject::Read::read(int64_t ofs, int64_t end, bufferlist& bl, RGWObjVersionTracker *objv_tracker) +int RGWRados::SystemObject::Read::read(int64_t ofs, int64_t end, bufferlist& bl, + RGWObjVersionTracker *objv_tracker, + boost::optional refresh_version) { RGWRados *store = source->get_store(); rgw_raw_obj& obj = source->get_obj(); - return store->get_system_obj(source->get_ctx(), state, objv_tracker, obj, bl, ofs, end, read_params.attrs, read_params.cache_info); + return store->get_system_obj(source->get_ctx(), state, objv_tracker, obj, bl, + ofs, end, read_params.attrs, + read_params.cache_info, refresh_version); } int RGWRados::SystemObject::Read::get_attr(const char *name, bufferlist& dest) @@ -11811,13 +11816,16 @@ int RGWRados::get_bucket_instance_info(RGWObjectCtx& obj_ctx, const rgw_bucket& int RGWRados::get_bucket_instance_from_oid(RGWObjectCtx& obj_ctx, const string& oid, RGWBucketInfo& info, real_time *pmtime, map *pattrs, - rgw_cache_entry_info *cache_info) + rgw_cache_entry_info *cache_info, + boost::optional refresh_version) { ldout(cct, 20) << "reading from " << get_zone_params().domain_root << ":" << oid << dendl; bufferlist epbl; - int ret = rgw_get_system_obj(this, obj_ctx, get_zone_params().domain_root, oid, epbl, &info.objv_tracker, pmtime, pattrs, cache_info); + int ret = rgw_get_system_obj(this, obj_ctx, get_zone_params().domain_root, + oid, epbl, &info.objv_tracker, pmtime, pattrs, + cache_info, refresh_version); if (ret < 0) { return ret; } @@ -11840,13 +11848,16 @@ int RGWRados::get_bucket_entrypoint_info(RGWObjectCtx& obj_ctx, RGWObjVersionTracker *objv_tracker, real_time *pmtime, map *pattrs, - rgw_cache_entry_info *cache_info) + rgw_cache_entry_info *cache_info, + boost::optional refresh_version) { bufferlist bl; string bucket_entry; rgw_make_bucket_entry_name(tenant_name, bucket_name, bucket_entry); - int ret = rgw_get_system_obj(this, obj_ctx, get_zone_params().domain_root, bucket_entry, bl, objv_tracker, pmtime, pattrs, cache_info); + int ret = rgw_get_system_obj(this, obj_ctx, get_zone_params().domain_root, + bucket_entry, bl, objv_tracker, pmtime, pattrs, + cache_info, refresh_version); if (ret < 0) { return ret; } @@ -11933,7 +11944,9 @@ int RGWRados::_get_bucket_info(RGWObjectCtx& obj_ctx, real_time ep_mtime; RGWObjVersionTracker ot; rgw_cache_entry_info entry_cache_info; - int ret = get_bucket_entrypoint_info(obj_ctx, tenant, bucket_name, entry_point, &ot, &ep_mtime, pattrs, &entry_cache_info); + int ret = get_bucket_entrypoint_info(obj_ctx, tenant, bucket_name, + entry_point, &ot, &ep_mtime, pattrs, + &entry_cache_info, refresh_version); if (ret < 0) { /* only init these fields */ info.bucket.tenant = tenant; @@ -11967,7 +11980,8 @@ int RGWRados::_get_bucket_info(RGWObjectCtx& obj_ctx, rgw_cache_entry_info cache_info; - ret = get_bucket_instance_from_oid(obj_ctx, oid, e.info, &e.mtime, &e.attrs, &cache_info); + ret = get_bucket_instance_from_oid(obj_ctx, oid, e.info, &e.mtime, &e.attrs, + &cache_info, refresh_version); e.info.ep_objv = ot.read_version; info = e.info; if (ret < 0) { diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index af911d8ba7d7..2c78a2590e48 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -2702,7 +2702,8 @@ public: explicit Read(RGWRados::SystemObject *_source) : source(_source) {} int stat(RGWObjVersionTracker *objv_tracker); - int read(int64_t ofs, int64_t end, bufferlist& bl, RGWObjVersionTracker *objv_tracker); + int read(int64_t ofs, int64_t end, bufferlist& bl, RGWObjVersionTracker *objv_tracker, + boost::optional refresh_version = boost::none); int get_attr(const char *name, bufferlist& dest); }; }; @@ -3291,7 +3292,9 @@ public: RGWObjVersionTracker *objv_tracker, rgw_raw_obj& obj, bufferlist& bl, off_t ofs, off_t end, map *attrs, - rgw_cache_entry_info *cache_info); + rgw_cache_entry_info *cache_info, + boost::optional refresh_version = + boost::none); virtual void register_chained_cache(RGWChainedCache *cache) {} virtual bool chain_cache_entry(std::initializer_list cache_info_entries, @@ -3409,11 +3412,13 @@ public: int put_bucket_instance_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, map *pattrs); int get_bucket_entrypoint_info(RGWObjectCtx& obj_ctx, const string& tenant_name, const string& bucket_name, RGWBucketEntryPoint& entry_point, RGWObjVersionTracker *objv_tracker, - ceph::real_time *pmtime, map *pattrs, rgw_cache_entry_info *cache_info = NULL); + ceph::real_time *pmtime, map *pattrs, rgw_cache_entry_info *cache_info = NULL, + boost::optional refresh_version = boost::none); int get_bucket_instance_info(RGWObjectCtx& obj_ctx, const string& meta_key, RGWBucketInfo& info, ceph::real_time *pmtime, map *pattrs); int get_bucket_instance_info(RGWObjectCtx& obj_ctx, const rgw_bucket& bucket, RGWBucketInfo& info, ceph::real_time *pmtime, map *pattrs); int get_bucket_instance_from_oid(RGWObjectCtx& obj_ctx, const string& oid, RGWBucketInfo& info, ceph::real_time *pmtime, map *pattrs, - rgw_cache_entry_info *cache_info = NULL); + rgw_cache_entry_info *cache_info = NULL, + boost::optional refresh_version = boost::none); int convert_old_bucket_info(RGWObjectCtx& obj_ctx, const string& tenant_name, const string& bucket_name); static void make_bucket_entry_name(const string& tenant_name, const string& bucket_name, string& bucket_entry); diff --git a/src/rgw/rgw_tools.cc b/src/rgw/rgw_tools.cc index b79fecb6a856..088fcc622e3c 100644 --- a/src/rgw/rgw_tools.cc +++ b/src/rgw/rgw_tools.cc @@ -40,7 +40,7 @@ int rgw_put_system_obj(RGWRados *rgwstore, const rgw_pool& pool, const string& o int rgw_get_system_obj(RGWRados *rgwstore, RGWObjectCtx& obj_ctx, const rgw_pool& pool, const string& key, bufferlist& bl, RGWObjVersionTracker *objv_tracker, real_time *pmtime, map *pattrs, - rgw_cache_entry_info *cache_info) + rgw_cache_entry_info *cache_info, boost::optional refresh_version) { bufferlist::iterator iter; int request_len = READ_CHUNK_LEN; @@ -64,7 +64,7 @@ int rgw_get_system_obj(RGWRados *rgwstore, RGWObjectCtx& obj_ctx, const rgw_pool rop.read_params.cache_info = cache_info; - ret = rop.read(0, request_len - 1, bl, objv_tracker); + ret = rop.read(0, request_len - 1, bl, objv_tracker, refresh_version); if (ret == -ECANCELED) { /* raced, restart */ if (!original_readv.empty()) { diff --git a/src/rgw/rgw_tools.h b/src/rgw/rgw_tools.h index caae06f2960e..00f404a6d1ca 100644 --- a/src/rgw/rgw_tools.h +++ b/src/rgw/rgw_tools.h @@ -20,7 +20,8 @@ int rgw_put_system_obj(RGWRados *rgwstore, const rgw_pool& pool, const string& o RGWObjVersionTracker *objv_tracker, real_time set_mtime, map *pattrs = NULL); int rgw_get_system_obj(RGWRados *rgwstore, RGWObjectCtx& obj_ctx, const rgw_pool& pool, const string& key, bufferlist& bl, RGWObjVersionTracker *objv_tracker, real_time *pmtime, map *pattrs = NULL, - rgw_cache_entry_info *cache_info = NULL); + rgw_cache_entry_info *cache_info = NULL, + boost::optional refresh_version = boost::none); int rgw_delete_system_obj(RGWRados *rgwstore, const rgw_pool& pool, const string& oid, RGWObjVersionTracker *objv_tracker); -- 2.47.3