From 0b8faf8a5d91b13631e79bc5d948e3a086693cf7 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Wed, 5 Jun 2019 17:33:50 -0700 Subject: [PATCH] rgw: RGWRados::lock_exclusive, ::unlock now being handled at svc.cls Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_mdlog.h | 19 ++++++++--- src/rgw/rgw_metadata.cc | 31 ++++++++--------- src/rgw/rgw_rados.cc | 64 ----------------------------------- src/rgw/rgw_rados.h | 3 -- src/rgw/rgw_service.cc | 2 +- src/rgw/rgw_tools.cc | 27 +++++++++++++++ src/rgw/rgw_tools.h | 4 +++ src/rgw/services/svc_cls.cc | 46 +++++++++++++++++++++++++ src/rgw/services/svc_cls.h | 20 ++++++++++- src/rgw/services/svc_mdlog.cc | 11 +++--- src/rgw/services/svc_mdlog.h | 8 ++--- src/rgw/services/svc_rados.h | 2 +- 12 files changed, 137 insertions(+), 100 deletions(-) diff --git a/src/rgw/rgw_mdlog.h b/src/rgw/rgw_mdlog.h index 3cf4ea1ec29..ac04ccb6a7c 100644 --- a/src/rgw/rgw_mdlog.h +++ b/src/rgw/rgw_mdlog.h @@ -64,9 +64,13 @@ class RGWMetadataLogInfoCompletion : public RefCountedObject { class RGWMetadataLog { CephContext *cct; - RGWRados *store; const string prefix; + struct Svc { + RGWSI_Zone *zone{nullptr}; + RGWSI_Cls *cls{nullptr}; + } svc; + static std::string make_prefix(const std::string& period) { if (period.empty()) return META_LOG_OBJ_PREFIX; @@ -78,10 +82,17 @@ class RGWMetadataLog { void mark_modified(int shard_id); public: - RGWMetadataLog(CephContext *_cct, RGWRados *_store, const std::string& period) - : cct(_cct), store(_store), + RGWMetadataLog(CephContext *_cct, + RGWSI_Zone *_zone_svc, + RGWSI_Cls *_cls_svc, + const std::string& period) + : cct(_cct), prefix(make_prefix(period)), - lock("RGWMetaLog::lock") {} + lock("RGWMetaLog::lock") { + svc.zone = _zone_svc; + svc.cls = _cls_svc; + } + void get_shard_oid(int id, string& oid) const { char buf[16]; diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc index cba07e758fc..450de3c4092 100644 --- a/src/rgw/rgw_metadata.cc +++ b/src/rgw/rgw_metadata.cc @@ -8,7 +8,6 @@ #include "rgw_coroutine.h" #include "cls/version/cls_version_types.h" -#include "rgw_rados.h" #include "rgw_zone.h" #include "rgw_tools.h" #include "rgw_mdlog.h" @@ -105,16 +104,16 @@ void RGWMetadataLogData::decode_json(JSONObj *obj) { int RGWMetadataLog::add_entry(const string& hash_key, const string& section, const string& key, bufferlist& bl) { - if (!store->svc.zone->need_to_log_metadata()) + if (!svc.zone->need_to_log_metadata()) return 0; string oid; int shard_id; - store->shard_name(prefix, cct->_conf->rgw_md_log_max_shards, hash_key, oid, &shard_id); + rgw_shard_name(prefix, cct->_conf->rgw_md_log_max_shards, hash_key, oid, &shard_id); mark_modified(shard_id); real_time now = real_clock::now(); - return store->svc.cls->timelog.add(oid, now, section, key, bl, null_yield); + return svc.cls->timelog.add(oid, now, section, key, bl, null_yield); } int RGWMetadataLog::store_entries_in_shard(list& entries, int shard_id, librados::AioCompletion *completion) @@ -122,8 +121,8 @@ int RGWMetadataLog::store_entries_in_shard(list& entries, int sha string oid; mark_modified(shard_id); - store->shard_name(prefix, shard_id, oid); - return store->svc.cls->timelog.add(oid, entries, completion, false, null_yield); + rgw_shard_name(prefix, shard_id, oid); + return svc.cls->timelog.add(oid, entries, completion, false, null_yield); } void RGWMetadataLog::init_list_entries(int shard_id, const real_time& from_time, const real_time& end_time, @@ -159,9 +158,9 @@ int RGWMetadataLog::list_entries(void *handle, } std::string next_marker; - int ret = store->svc.cls->timelog.list(ctx->cur_oid, ctx->from_time, ctx->end_time, - max_entries, entries, ctx->marker, - &next_marker, truncated, null_yield); + int ret = svc.cls->timelog.list(ctx->cur_oid, ctx->from_time, ctx->end_time, + max_entries, entries, ctx->marker, + &next_marker, truncated, null_yield); if ((ret < 0) && (ret != -ENOENT)) return ret; @@ -183,7 +182,7 @@ int RGWMetadataLog::get_info(int shard_id, RGWMetadataLogInfo *info) cls_log_header header; - int ret = store->svc.cls->timelog.info(oid, &header, null_yield); + int ret = svc.cls->timelog.info(oid, &header, null_yield); if ((ret < 0) && (ret != -ENOENT)) return ret; @@ -219,9 +218,9 @@ int RGWMetadataLog::get_info_async(int shard_id, RGWMetadataLogInfoCompletion *c completion->get(); // hold a ref until the completion fires - return store->svc.cls->timelog.info_async(completion->get_io_obj(), oid, - &completion->get_header(), - completion->get_completion()); + return svc.cls->timelog.info_async(completion->get_io_obj(), oid, + &completion->get_header(), + completion->get_completion()); } int RGWMetadataLog::trim(int shard_id, const real_time& from_time, const real_time& end_time, @@ -232,7 +231,7 @@ int RGWMetadataLog::trim(int shard_id, const real_time& from_time, const real_ti int ret; - ret = store->svc.cls->timelog.trim(oid, from_time, end_time, start_marker, end_marker, nullptr, null_yield); + ret = svc.cls->timelog.trim(oid, from_time, end_time, start_marker, end_marker, nullptr, null_yield); if (ret == -ENOENT || ret == -ENODATA) ret = 0; @@ -244,14 +243,14 @@ int RGWMetadataLog::lock_exclusive(int shard_id, timespan duration, string& zone string oid; get_shard_oid(shard_id, oid); - return store->lock_exclusive(store->svc.zone->get_zone_params().log_pool, oid, duration, zone_id, owner_id); + return svc.cls->lock.lock_exclusive(svc.zone->get_zone_params().log_pool, oid, duration, zone_id, owner_id); } int RGWMetadataLog::unlock(int shard_id, string& zone_id, string& owner_id) { string oid; get_shard_oid(shard_id, oid); - return store->unlock(store->svc.zone->get_zone_params().log_pool, oid, zone_id, owner_id); + return svc.cls->lock.unlock(svc.zone->get_zone_params().log_pool, oid, zone_id, owner_id); } void RGWMetadataLog::mark_modified(int shard_id) diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 73981e81888..2018b566a7f 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -102,7 +102,6 @@ static string shadow_ns = "shadow"; static string default_bucket_index_pool_suffix = "rgw.buckets.index"; static string default_storage_extra_pool_suffix = "rgw.buckets.non-ec"; -static string log_lock_name = "rgw_log_lock"; static RGWObjCategory main_category = RGWObjCategory::Main; #define RGW_USAGE_OBJ_PREFIX "usage." @@ -1684,69 +1683,6 @@ int RGWRados::clear_usage() return ret; } -void RGWRados::shard_name(const string& prefix, unsigned max_shards, const string& key, string& name, int *shard_id) -{ - uint32_t val = ceph_str_hash_linux(key.c_str(), key.size()); - char buf[16]; - if (shard_id) { - *shard_id = val % max_shards; - } - snprintf(buf, sizeof(buf), "%u", (unsigned)(val % max_shards)); - name = prefix + buf; -} - -void RGWRados::shard_name(const string& prefix, unsigned max_shards, const string& section, const string& key, string& name) -{ - uint32_t val = ceph_str_hash_linux(key.c_str(), key.size()); - val ^= ceph_str_hash_linux(section.c_str(), section.size()); - char buf[16]; - snprintf(buf, sizeof(buf), "%u", (unsigned)(val % max_shards)); - name = prefix + buf; -} - -void RGWRados::shard_name(const string& prefix, unsigned shard_id, string& name) -{ - char buf[16]; - snprintf(buf, sizeof(buf), "%u", shard_id); - name = prefix + buf; - -} - -int RGWRados::lock_exclusive(const rgw_pool& pool, const string& oid, timespan& duration, - string& zone_id, string& owner_id) { - librados::IoCtx io_ctx; - - int r = rgw_init_ioctx(get_rados_handle(), pool, io_ctx); - if (r < 0) { - return r; - } - uint64_t msec = std::chrono::duration_cast(duration).count(); - utime_t ut(msec / 1000, msec % 1000); - - rados::cls::lock::Lock l(log_lock_name); - l.set_duration(ut); - l.set_cookie(owner_id); - l.set_tag(zone_id); - l.set_may_renew(true); - - return l.lock_exclusive(&io_ctx, oid); -} - -int RGWRados::unlock(const rgw_pool& pool, const string& oid, string& zone_id, string& owner_id) { - librados::IoCtx io_ctx; - - int r = rgw_init_ioctx(get_rados_handle(), pool, io_ctx); - if (r < 0) { - return r; - } - - rados::cls::lock::Lock l(log_lock_name); - l.set_tag(zone_id); - l.set_cookie(owner_id); - - return l.unlock(&io_ctx, oid); -} - int RGWRados::decode_policy(bufferlist& bl, ACLOwner *owner) { auto i = bl.cbegin(); diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 0246574dfae..48531dbb377 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1396,9 +1396,6 @@ public: uint64_t end_epoch); int cls_obj_usage_log_clear(string& oid); - void shard_name(const string& prefix, unsigned max_shards, const string& key, string& name, int *shard_id); - void shard_name(const string& prefix, unsigned max_shards, const string& section, const string& key, string& name); - void shard_name(const string& prefix, unsigned shard_id, string& name); int get_target_shard_id(const RGWBucketInfo& bucket_info, const string& obj_key, int *shard_id); int lock_exclusive(const rgw_pool& pool, const string& oid, ceph::timespan& duration, string& zone_id, string& owner_id); diff --git a/src/rgw/rgw_service.cc b/src/rgw/rgw_service.cc index b0e19d2a379..e5317065630 100644 --- a/src/rgw/rgw_service.cc +++ b/src/rgw/rgw_service.cc @@ -73,7 +73,7 @@ int RGWServices_Def::init(CephContext *cct, bi_rados.get(), meta.get(), meta_be_sobj.get(), sync_modules.get()); cls->init(zone.get(), rados.get()); - mdlog->init(rados.get(), zone.get(), sysobj.get()); + mdlog->init(rados.get(), zone.get(), sysobj.get(), cls.get()); meta->init(sysobj.get(), mdlog.get(), meta_bes); meta_be_sobj->init(sysobj.get(), mdlog.get()); meta_be_otp->init(sysobj.get(), mdlog.get(), cls.get()); diff --git a/src/rgw/rgw_tools.cc b/src/rgw/rgw_tools.cc index 703025897a1..a6b8d8a1c35 100644 --- a/src/rgw/rgw_tools.cc +++ b/src/rgw/rgw_tools.cc @@ -96,6 +96,33 @@ int rgw_init_ioctx(librados::Rados *rados, const rgw_pool& pool, return 0; } +void rgw_shard_name(const string& prefix, unsigned max_shards, const string& key, string& name, int *shard_id) +{ + uint32_t val = ceph_str_hash_linux(key.c_str(), key.size()); + char buf[16]; + if (shard_id) { + *shard_id = val % max_shards; + } + snprintf(buf, sizeof(buf), "%u", (unsigned)(val % max_shards)); + name = prefix + buf; +} + +void rgw_shard_name(const string& prefix, unsigned max_shards, const string& section, const string& key, string& name) +{ + uint32_t val = ceph_str_hash_linux(key.c_str(), key.size()); + val ^= ceph_str_hash_linux(section.c_str(), section.size()); + char buf[16]; + snprintf(buf, sizeof(buf), "%u", (unsigned)(val % max_shards)); + name = prefix + buf; +} + +void rgw_shard_name(const string& prefix, unsigned shard_id, string& name) +{ + char buf[16]; + snprintf(buf, sizeof(buf), "%u", shard_id); + name = prefix + buf; +} + int rgw_put_system_obj(RGWSysObjectCtx& obj_ctx, const rgw_pool& pool, const string& oid, bufferlist& data, bool exclusive, RGWObjVersionTracker *objv_tracker, real_time set_mtime, map *pattrs) { diff --git a/src/rgw/rgw_tools.h b/src/rgw/rgw_tools.h index e4e5b6daebd..89fa696927b 100644 --- a/src/rgw/rgw_tools.h +++ b/src/rgw/rgw_tools.h @@ -54,6 +54,10 @@ static inline int rgw_shard_id(const string& key, int max_shards) max_shards); } +void rgw_shard_name(const string& prefix, unsigned max_shards, const string& key, string& name, int *shard_id); +void rgw_shard_name(const string& prefix, unsigned max_shards, const string& section, const string& key, string& name); +void rgw_shard_name(const string& prefix, unsigned shard_id, string& name); + int rgw_put_system_obj(RGWSysObjectCtx& obj_ctx, const rgw_pool& pool, const string& oid, bufferlist& data, bool exclusive, RGWObjVersionTracker *objv_tracker, real_time set_mtime, map *pattrs = NULL); int rgw_get_system_obj(RGWSysObjectCtx& obj_ctx, const rgw_pool& pool, const string& key, bufferlist& bl, diff --git a/src/rgw/services/svc_cls.cc b/src/rgw/services/svc_cls.cc index 0e33a6210aa..3ac0ed06713 100644 --- a/src/rgw/services/svc_cls.cc +++ b/src/rgw/services/svc_cls.cc @@ -8,10 +8,12 @@ #include "cls/otp/cls_otp_client.h" #include "cls/log/cls_log_client.h" +#include "cls/lock/cls_lock_client.h" #define dout_subsys ceph_subsys_rgw +static string log_lock_name = "rgw_log_lock"; int RGWSI_Cls::do_start() { @@ -417,3 +419,47 @@ int RGWSI_Cls::TimeLog::trim(const string& oid, return r; } +int RGWSI_Cls::Lock::lock_exclusive(const rgw_pool& pool, + const string& oid, + timespan& duration, + string& zone_id, + string& owner_id, + std::optional lock_name) +{ + auto p = rados_svc->pool(pool); + int r = p.open(); + if (r < 0) { + return r; + } + + uint64_t msec = std::chrono::duration_cast(duration).count(); + utime_t ut(msec / 1000, msec % 1000); + + rados::cls::lock::Lock l(lock_name.value_or(log_lock_name)); + l.set_duration(ut); + l.set_cookie(owner_id); + l.set_tag(zone_id); + l.set_may_renew(true); + + return l.lock_exclusive(&p.ioctx(), oid); +} + +int RGWSI_Cls::Lock::unlock(const rgw_pool& pool, + const string& oid, + string& zone_id, + string& owner_id, + std::optional lock_name) +{ + auto p = rados_svc->pool(pool); + int r = p.open(); + if (r < 0) { + return r; + } + + rados::cls::lock::Lock l(lock_name.value_or(log_lock_name)); + l.set_tag(zone_id); + l.set_cookie(owner_id); + + return l.unlock(&p.ioctx(), oid); +} + diff --git a/src/rgw/services/svc_cls.h b/src/rgw/services/svc_cls.h index e973a59a4bc..9dbff5ce23e 100644 --- a/src/rgw/services/svc_cls.h +++ b/src/rgw/services/svc_cls.h @@ -124,7 +124,24 @@ public: optional_yield y); } timelog; - RGWSI_Cls(CephContext *cct): RGWServiceInstance(cct), mfa(cct), timelog(cct) {} + class Lock : public ClsSubService { + int init_obj(const string& oid, RGWSI_RADOS::Obj& obj); + public: + Lock(CephContext *cct): ClsSubService(cct) {} + int lock_exclusive(const rgw_pool& pool, + const string& oid, + timespan& duration, + string& zone_id, + string& owner_id, + std::optional lock_name = std::nullopt); + int unlock(const rgw_pool& pool, + const string& oid, + string& zone_id, + string& owner_id, + std::optional lock_name = std::nullopt); + } lock; + + RGWSI_Cls(CephContext *cct): RGWServiceInstance(cct), mfa(cct), timelog(cct), lock(cct) {} void init(RGWSI_Zone *_zone_svc, RGWSI_RADOS *_rados_svc) { rados_svc = _rados_svc; @@ -132,6 +149,7 @@ public: mfa.init(this, zone_svc, rados_svc); timelog.init(this, zone_svc, rados_svc); + lock.init(this, zone_svc, rados_svc); } int do_start() override; diff --git a/src/rgw/services/svc_mdlog.cc b/src/rgw/services/svc_mdlog.cc index 1ba9a97b93c..8e394aa0d5d 100644 --- a/src/rgw/services/svc_mdlog.cc +++ b/src/rgw/services/svc_mdlog.cc @@ -24,12 +24,13 @@ RGWSI_MDLog::RGWSI_MDLog(CephContext *cct) : RGWServiceInstance(cct) { RGWSI_MDLog::~RGWSI_MDLog() { } -int RGWSI_MDLog::init(RGWSI_RADOS *_rados_svc, RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc) +int RGWSI_MDLog::init(RGWSI_RADOS *_rados_svc, RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc, RGWSI_Cls *_cls_svc) { - svc.zone = zone_svc; - svc.sysobj = sysobj_svc; + svc.zone = _zone_svc; + svc.sysobj = _sysobj_svc; svc.mdlog = this; - svc.rados = rados_svc; + svc.rados = _rados_svc; + svc.cls = _cls_svc; return 0; } @@ -357,7 +358,7 @@ RGWMetadataLog* RGWSI_MDLog::get_log(const std::string& period) // construct the period's log in place if it doesn't exist auto insert = md_logs.emplace(std::piecewise_construct, std::forward_as_tuple(period), - std::forward_as_tuple(cct, store, period)); + std::forward_as_tuple(cct, svc.zone, svc.cls, period)); return &insert.first->second; } diff --git a/src/rgw/services/svc_mdlog.h b/src/rgw/services/svc_mdlog.h index bcd91d74c24..8a7c60bf1b1 100644 --- a/src/rgw/services/svc_mdlog.h +++ b/src/rgw/services/svc_mdlog.h @@ -41,10 +41,6 @@ class RGWSI_MDLog : public RGWServiceInstance friend class mdlog::ReadHistoryCR; friend class mdlog::WriteHistoryCR; - RGWSI_RADOS *rados_svc{nullptr}; - RGWSI_Zone *zone_svc{nullptr}; - RGWSI_SysObj *sysobj_svc{nullptr}; - // maintain a separate metadata log for each period std::map md_logs; @@ -68,11 +64,13 @@ public: RGWSI_Zone *zone{nullptr}; RGWSI_SysObj *sysobj{nullptr}; RGWSI_MDLog *mdlog{nullptr}; + RGWSI_Cls *cls{nullptr}; } svc; int init(RGWSI_RADOS *_rados_svc, RGWSI_Zone *_zone_svc, - RGWSI_SysObj *_sysobj_svc); + RGWSI_SysObj *_sysobj_svc, + RGWSI_Cls *_cls_svc); int do_start() override; diff --git a/src/rgw/services/svc_rados.h b/src/rgw/services/svc_rados.h index 00c58ee713c..14c78a86bec 100644 --- a/src/rgw/services/svc_rados.h +++ b/src/rgw/services/svc_rados.h @@ -34,7 +34,6 @@ class RGWSI_RADOS : public RGWServiceInstance std::unique_ptr async_processor; int do_start() override; - void shutdown() override; public: struct OpenParams { @@ -66,6 +65,7 @@ public: ~RGWSI_RADOS(); void init() {} + void shutdown() override; uint64_t instance_id(); -- 2.39.5