From 34864fe7ba839e0d593437b6e62f6812f1c1a375 Mon Sep 17 00:00:00 2001 From: "Adam C. Emerson" Date: Wed, 20 Dec 2017 17:06:32 -0500 Subject: [PATCH] rgw: Plumb refresh logic into object cache Now when we force a refetch of bucket info it will actually go to the OSD rather than simply using the objects in the object cache. Fixes: http://tracker.ceph.com/issues/22517 Signed-off-by: Adam C. Emerson (cherry picked from commit d997f657750faf920170843e62deacab70008d8b) --- src/rgw/rgw_cache.h | 11 +++++++---- src/rgw/rgw_rados.cc | 32 +++++++++++++++++++++++--------- src/rgw/rgw_rados.h | 13 +++++++++---- src/rgw/rgw_tools.cc | 4 ++-- src/rgw/rgw_tools.h | 3 ++- 5 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/rgw/rgw_cache.h b/src/rgw/rgw_cache.h index 1c03c271060..20c1b02287b 100644 --- a/src/rgw/rgw_cache.h +++ b/src/rgw/rgw_cache.h @@ -242,7 +242,8 @@ public: RGWObjVersionTracker *objv_tracker, rgw_raw_obj& obj, bufferlist& bl, off_t ofs, off_t end, map *attrs, - rgw_cache_entry_info *cache_info) override; + rgw_cache_entry_info *cache_info, + boost::optional refresh_version = boost::none) override; int raw_obj_stat(rgw_raw_obj& obj, uint64_t *psize, real_time *pmtime, uint64_t *epoch, map *attrs, bufferlist *first_chunk, RGWObjVersionTracker *objv_tracker) override; @@ -287,7 +288,8 @@ int RGWCache::get_system_obj(RGWObjectCtx& obj_ctx, RGWRados::SystemObject::R RGWObjVersionTracker *objv_tracker, rgw_raw_obj& obj, bufferlist& obl, off_t ofs, off_t end, map *attrs, - rgw_cache_entry_info *cache_info) + rgw_cache_entry_info *cache_info, + boost::optional refresh_version) { rgw_pool pool; string oid; @@ -304,8 +306,9 @@ int RGWCache::get_system_obj(RGWObjectCtx& obj_ctx, RGWRados::SystemObject::R flags |= CACHE_FLAG_OBJV; if (attrs) flags |= CACHE_FLAG_XATTRS; - - if (cache.get(name, info, flags, cache_info) == 0) { + + if ((cache.get(name, info, flags, cache_info) == 0) && + (!refresh_version || !info.version.compare(&(*refresh_version)))) { if (info.status < 0) return info.status; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index b7c63d3977c..3c46d6e698e 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -10236,7 +10236,8 @@ int RGWRados::get_system_obj(RGWObjectCtx& obj_ctx, RGWRados::SystemObject::Read RGWObjVersionTracker *objv_tracker, rgw_raw_obj& obj, bufferlist& bl, off_t ofs, off_t end, map *attrs, - rgw_cache_entry_info *cache_info) + rgw_cache_entry_info *cache_info, + boost::optional) { uint64_t len; ObjectReadOperation op; @@ -10283,12 +10284,16 @@ int RGWRados::get_system_obj(RGWObjectCtx& obj_ctx, RGWRados::SystemObject::Read return bl.length(); } -int RGWRados::SystemObject::Read::read(int64_t ofs, int64_t end, bufferlist& bl, RGWObjVersionTracker *objv_tracker) +int RGWRados::SystemObject::Read::read(int64_t ofs, int64_t end, bufferlist& bl, + RGWObjVersionTracker *objv_tracker, + boost::optional refresh_version) { RGWRados *store = source->get_store(); rgw_raw_obj& obj = source->get_obj(); - return store->get_system_obj(source->get_ctx(), state, objv_tracker, obj, bl, ofs, end, read_params.attrs, read_params.cache_info); + return store->get_system_obj(source->get_ctx(), state, objv_tracker, obj, bl, + ofs, end, read_params.attrs, + read_params.cache_info, refresh_version); } int RGWRados::SystemObject::Read::get_attr(const char *name, bufferlist& dest) @@ -11826,13 +11831,16 @@ int RGWRados::get_bucket_instance_info(RGWObjectCtx& obj_ctx, const rgw_bucket& int RGWRados::get_bucket_instance_from_oid(RGWObjectCtx& obj_ctx, const string& oid, RGWBucketInfo& info, real_time *pmtime, map *pattrs, - rgw_cache_entry_info *cache_info) + rgw_cache_entry_info *cache_info, + boost::optional refresh_version) { ldout(cct, 20) << "reading from " << get_zone_params().domain_root << ":" << oid << dendl; bufferlist epbl; - int ret = rgw_get_system_obj(this, obj_ctx, get_zone_params().domain_root, oid, epbl, &info.objv_tracker, pmtime, pattrs, cache_info); + int ret = rgw_get_system_obj(this, obj_ctx, get_zone_params().domain_root, + oid, epbl, &info.objv_tracker, pmtime, pattrs, + cache_info, refresh_version); if (ret < 0) { return ret; } @@ -11855,13 +11863,16 @@ int RGWRados::get_bucket_entrypoint_info(RGWObjectCtx& obj_ctx, RGWObjVersionTracker *objv_tracker, real_time *pmtime, map *pattrs, - rgw_cache_entry_info *cache_info) + rgw_cache_entry_info *cache_info, + boost::optional refresh_version) { bufferlist bl; string bucket_entry; rgw_make_bucket_entry_name(tenant_name, bucket_name, bucket_entry); - int ret = rgw_get_system_obj(this, obj_ctx, get_zone_params().domain_root, bucket_entry, bl, objv_tracker, pmtime, pattrs, cache_info); + int ret = rgw_get_system_obj(this, obj_ctx, get_zone_params().domain_root, + bucket_entry, bl, objv_tracker, pmtime, pattrs, + cache_info, refresh_version); if (ret < 0) { return ret; } @@ -11947,7 +11958,9 @@ int RGWRados::_get_bucket_info(RGWObjectCtx& obj_ctx, real_time ep_mtime; RGWObjVersionTracker ot; rgw_cache_entry_info entry_cache_info; - int ret = get_bucket_entrypoint_info(obj_ctx, tenant, bucket_name, entry_point, &ot, &ep_mtime, pattrs, &entry_cache_info); + int ret = get_bucket_entrypoint_info(obj_ctx, tenant, bucket_name, + entry_point, &ot, &ep_mtime, pattrs, + &entry_cache_info, refresh_version); if (ret < 0) { /* only init these fields */ info.bucket.tenant = tenant; @@ -11981,7 +11994,8 @@ int RGWRados::_get_bucket_info(RGWObjectCtx& obj_ctx, rgw_cache_entry_info cache_info; - ret = get_bucket_instance_from_oid(obj_ctx, oid, e.info, &e.mtime, &e.attrs, &cache_info); + ret = get_bucket_instance_from_oid(obj_ctx, oid, e.info, &e.mtime, &e.attrs, + &cache_info, refresh_version); e.info.ep_objv = ot.read_version; info = e.info; if (ret < 0) { diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 24d94a7306c..8354c8c42d8 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -2676,7 +2676,8 @@ public: explicit Read(RGWRados::SystemObject *_source) : source(_source) {} int stat(RGWObjVersionTracker *objv_tracker); - int read(int64_t ofs, int64_t end, bufferlist& bl, RGWObjVersionTracker *objv_tracker); + int read(int64_t ofs, int64_t end, bufferlist& bl, RGWObjVersionTracker *objv_tracker, + boost::optional refresh_version = boost::none); int get_attr(const char *name, bufferlist& dest); }; }; @@ -3268,7 +3269,9 @@ public: RGWObjVersionTracker *objv_tracker, rgw_raw_obj& obj, bufferlist& bl, off_t ofs, off_t end, map *attrs, - rgw_cache_entry_info *cache_info); + rgw_cache_entry_info *cache_info, + boost::optional refresh_version = + boost::none); virtual void register_chained_cache(RGWChainedCache *cache) {} virtual bool chain_cache_entry(list& cache_info_entries, RGWChainedCache::Entry *chained_entry) { return false; } @@ -3385,11 +3388,13 @@ public: int put_bucket_instance_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, map *pattrs); int get_bucket_entrypoint_info(RGWObjectCtx& obj_ctx, const string& tenant_name, const string& bucket_name, RGWBucketEntryPoint& entry_point, RGWObjVersionTracker *objv_tracker, - ceph::real_time *pmtime, map *pattrs, rgw_cache_entry_info *cache_info = NULL); + ceph::real_time *pmtime, map *pattrs, rgw_cache_entry_info *cache_info = NULL, + boost::optional refresh_version = boost::none); int get_bucket_instance_info(RGWObjectCtx& obj_ctx, const string& meta_key, RGWBucketInfo& info, ceph::real_time *pmtime, map *pattrs); int get_bucket_instance_info(RGWObjectCtx& obj_ctx, const rgw_bucket& bucket, RGWBucketInfo& info, ceph::real_time *pmtime, map *pattrs); int get_bucket_instance_from_oid(RGWObjectCtx& obj_ctx, const string& oid, RGWBucketInfo& info, ceph::real_time *pmtime, map *pattrs, - rgw_cache_entry_info *cache_info = NULL); + rgw_cache_entry_info *cache_info = NULL, + boost::optional refresh_version = boost::none); int convert_old_bucket_info(RGWObjectCtx& obj_ctx, const string& tenant_name, const string& bucket_name); static void make_bucket_entry_name(const string& tenant_name, const string& bucket_name, string& bucket_entry); diff --git a/src/rgw/rgw_tools.cc b/src/rgw/rgw_tools.cc index b79fecb6a85..088fcc622e3 100644 --- a/src/rgw/rgw_tools.cc +++ b/src/rgw/rgw_tools.cc @@ -40,7 +40,7 @@ int rgw_put_system_obj(RGWRados *rgwstore, const rgw_pool& pool, const string& o int rgw_get_system_obj(RGWRados *rgwstore, RGWObjectCtx& obj_ctx, const rgw_pool& pool, const string& key, bufferlist& bl, RGWObjVersionTracker *objv_tracker, real_time *pmtime, map *pattrs, - rgw_cache_entry_info *cache_info) + rgw_cache_entry_info *cache_info, boost::optional refresh_version) { bufferlist::iterator iter; int request_len = READ_CHUNK_LEN; @@ -64,7 +64,7 @@ int rgw_get_system_obj(RGWRados *rgwstore, RGWObjectCtx& obj_ctx, const rgw_pool rop.read_params.cache_info = cache_info; - ret = rop.read(0, request_len - 1, bl, objv_tracker); + ret = rop.read(0, request_len - 1, bl, objv_tracker, refresh_version); if (ret == -ECANCELED) { /* raced, restart */ if (!original_readv.empty()) { diff --git a/src/rgw/rgw_tools.h b/src/rgw/rgw_tools.h index caae06f2960..00f404a6d1c 100644 --- a/src/rgw/rgw_tools.h +++ b/src/rgw/rgw_tools.h @@ -20,7 +20,8 @@ int rgw_put_system_obj(RGWRados *rgwstore, const rgw_pool& pool, const string& o RGWObjVersionTracker *objv_tracker, real_time set_mtime, map *pattrs = NULL); int rgw_get_system_obj(RGWRados *rgwstore, RGWObjectCtx& obj_ctx, const rgw_pool& pool, const string& key, bufferlist& bl, RGWObjVersionTracker *objv_tracker, real_time *pmtime, map *pattrs = NULL, - rgw_cache_entry_info *cache_info = NULL); + rgw_cache_entry_info *cache_info = NULL, + boost::optional refresh_version = boost::none); int rgw_delete_system_obj(RGWRados *rgwstore, const rgw_pool& pool, const string& oid, RGWObjVersionTracker *objv_tracker); -- 2.47.3