From 9073524d5cbef69eebde0ac5b335d23b45a3f0bf Mon Sep 17 00:00:00 2001 From: Shilpa Jagannath Date: Thu, 15 Oct 2020 13:14:52 +0530 Subject: [PATCH] rgw: don't allow overwrites to BucketInfo's "layout" when sync fetches bucket metadata. To support dynamic resharding with multisite, each zone needs to have its own view of "layout" to be able to manage resharding decisions independently. Signed-off-by: Shilpa Jagannath --- src/rgw/rgw_admin.cc | 2 +- src/rgw/rgw_bucket.cc | 43 +++++++++++++++++++++++++++++------------ src/rgw/rgw_bucket.h | 2 ++ src/rgw/rgw_metadata.cc | 17 +++++++++------- src/rgw/rgw_metadata.h | 17 +++++++++++----- src/rgw/rgw_otp.cc | 2 +- src/rgw/rgw_rados.cc | 7 +++---- src/rgw/rgw_sync.cc | 2 +- src/rgw/rgw_user.cc | 8 ++++---- 9 files changed, 65 insertions(+), 35 deletions(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index b32561867256..abdf94180968 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -7476,7 +7476,7 @@ next: cerr << "ERROR: failed to read input: " << cpp_strerror(-ret) << std::endl; return -ret; } - ret = store->ctl()->meta.mgr->put(metadata_key, bl, null_yield, RGWMDLogSyncType::APPLY_ALWAYS); + ret = store->ctl()->meta.mgr->put(metadata_key, bl, null_yield, RGWMDLogSyncType::APPLY_ALWAYS, false); if (ret < 0) { cerr << "ERROR: can't put key: " << cpp_strerror(-ret) << std::endl; return -ret; diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 584829919e8b..f412f40360eb 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -2041,7 +2041,7 @@ public: RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType type) override; + RGWMDLogSyncType type, bool from_remote_zone) override; int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, optional_yield y) override { @@ -2095,7 +2095,7 @@ public: RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType type) : RGWMetadataHandlerPut_SObj(_handler, op, entry, obj, objv_tracker, y, type), + RGWMDLogSyncType type, bool from_remote_zone) : RGWMetadataHandlerPut_SObj(_handler, op, entry, obj, objv_tracker, y, type, from_remote_zone), bhandler(_handler) { obj = static_cast(_obj); } @@ -2113,9 +2113,9 @@ int RGWBucketMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op *op, string& RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType type) + RGWMDLogSyncType type, bool from_remote_zone) { - RGWMetadataHandlerPut_Bucket put_op(this, op, entry, obj, objv_tracker, y, type); + RGWMetadataHandlerPut_Bucket put_op(this, op, entry, obj, objv_tracker, y, type, from_remote_zone); return do_put_operate(&put_op); } @@ -2358,7 +2358,7 @@ public: RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType type) override { + RGWMDLogSyncType type, bool from_remote_zone) override { if (entry.find("-deleted-") != string::npos) { RGWObjVersionTracker ot; RGWMetadataObject *robj; @@ -2378,7 +2378,7 @@ public: } return RGWBucketMetadataHandler::do_put(op, entry, obj, - objv_tracker, y, type); + objv_tracker, y, type, from_remote_zone); } }; @@ -2449,7 +2449,7 @@ public: int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType sync_type) override; + RGWMDLogSyncType sync_type, bool from_remote_zone) override; int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, optional_yield y) override { @@ -2483,13 +2483,13 @@ class RGWMetadataHandlerPut_BucketInstance : public RGWMetadataHandlerPut_SObj RGWBucketInstanceMetadataHandler *bihandler; RGWBucketInstanceMetadataObject *obj; public: - RGWMetadataHandlerPut_BucketInstance(CephContext *cct, + RGWMetadataHandlerPut_BucketInstance(CephContext *_cct, RGWBucketInstanceMetadataHandler *_handler, RGWSI_MetaBackend_Handler::Op *_op, string& entry, RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType type) : RGWMetadataHandlerPut_SObj(_handler, _op, entry, obj, objv_tracker, y, type), - bihandler(_handler) { + RGWMDLogSyncType type, bool from_remote_zone) : RGWMetadataHandlerPut_SObj(_handler, _op, entry, obj, objv_tracker, y, type, from_remote_zone), + cct(_cct), bihandler(_handler) { obj = static_cast(_obj); auto& bci = obj->get_bci(); @@ -2510,13 +2510,23 @@ int RGWBucketInstanceMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op *op, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType type) + RGWMDLogSyncType type, bool from_remote_zone) { RGWMetadataHandlerPut_BucketInstance put_op(svc.bucket->ctx(), this, op, entry, obj, - objv_tracker, y, type); + objv_tracker, y, type, from_remote_zone); return do_put_operate(&put_op); } +void init_default_bucket_layout(CephContext *cct, RGWBucketInfo& info, const RGWZone& zone) { + info.layout.current_index.gen = 0; + info.layout.current_index.layout.normal.hash_type = rgw::BucketHashType::Mod; + info.layout.current_index.layout.type = rgw::BucketIndexType::Normal; + + info.layout.current_index.layout.normal.num_shards = ( + cct->_conf->rgw_override_bucket_index_max_shards > 0 ? + cct->_conf->rgw_override_bucket_index_max_shards : zone.bucket_index_max_shards); +} + int RGWMetadataHandlerPut_BucketInstance::put_check() { int ret; @@ -2529,6 +2539,15 @@ int RGWMetadataHandlerPut_BucketInstance::put_check() bool exists = (!!orig_obj); + if (from_remote_zone) { + // don't sync bucket layout changes + if (!exists) { + init_default_bucket_layout(cct, bci.info, bihandler->svc.zone->get_zone()); + } else { + bci.info.layout = old_bci->info.layout; + } + } + if (!exists || old_bci->info.bucket.bucket_id != bci.info.bucket.bucket_id) { /* a new bucket, we need to select a new bucket placement for it */ string tenant_name; diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index cb93aefde74a..e12ba2670d4e 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -57,6 +57,8 @@ extern void rgw_parse_url_bucket(const string& bucket, // conforms to the type declaration of RGWRados::check_filter_t. extern bool rgw_bucket_object_check_filter(const string& oid); +extern void init_default_bucket_layout(CephContext *cct, RGWBucketInfo& info, const RGWZone& zone); + struct RGWBucketCompleteInfo { RGWBucketInfo info; map attrs; diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc index 713f6b780f27..be572ebc866a 100644 --- a/src/rgw/rgw_metadata.cc +++ b/src/rgw/rgw_metadata.cc @@ -311,7 +311,7 @@ public: } int put(string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, - optional_yield y, RGWMDLogSyncType type) override { + optional_yield y, RGWMDLogSyncType type, bool from_remote_zone) override { return -ENOTSUP; } @@ -388,19 +388,20 @@ RGWMetadataHandler_GenericMetaBE::Put::Put(RGWMetadataHandler_GenericMetaBE *_ha string& _entry, RGWMetadataObject *_obj, RGWObjVersionTracker& _objv_tracker, optional_yield _y, - RGWMDLogSyncType _type): + RGWMDLogSyncType _type, bool _from_remote_zone): handler(_handler), op(_op), entry(_entry), obj(_obj), objv_tracker(_objv_tracker), apply_type(_type), - y(_y) + y(_y), + from_remote_zone(_from_remote_zone) { } RGWMetadataHandlerPut_SObj::RGWMetadataHandlerPut_SObj(RGWMetadataHandler_GenericMetaBE *handler, RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType type) : Put(handler, op, entry, obj, objv_tracker, y, type) { + RGWMDLogSyncType type, bool from_remote_zone) : Put(handler, op, entry, obj, objv_tracker, y, type, from_remote_zone) { } RGWMetadataHandlerPut_SObj::~RGWMetadataHandlerPut_SObj() { @@ -482,10 +483,11 @@ int RGWMetadataHandler_GenericMetaBE::get(string& entry, RGWMetadataObject **obj }); } -int RGWMetadataHandler_GenericMetaBE::put(string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield y, RGWMDLogSyncType type) +int RGWMetadataHandler_GenericMetaBE::put(string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, + optional_yield y, RGWMDLogSyncType type, bool from_remote_zone) { return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - return do_put(op, entry, obj, objv_tracker, y, type); + return do_put(op, entry, obj, objv_tracker, y, type, from_remote_zone); }); } @@ -659,6 +661,7 @@ int RGWMetadataManager::get(string& metadata_key, Formatter *f, optional_yield y int RGWMetadataManager::put(string& metadata_key, bufferlist& bl, optional_yield y, RGWMDLogSyncType sync_type, + bool from_remote_zone, obj_version *existing_version) { RGWMetadataHandler *handler; @@ -698,7 +701,7 @@ int RGWMetadataManager::put(string& metadata_key, bufferlist& bl, return -EINVAL; } - ret = handler->put(entry, obj, objv_tracker, y, sync_type); + ret = handler->put(entry, obj, objv_tracker, y, sync_type, from_remote_zone); if (existing_version) { *existing_version = objv_tracker.read_version; } diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h index 1bc2ebef8333..97d4f0487606 100644 --- a/src/rgw/rgw_metadata.h +++ b/src/rgw/rgw_metadata.h @@ -70,7 +70,11 @@ public: virtual RGWMetadataObject *get_meta_obj(JSONObj *jo, const obj_version& objv, const ceph::real_time& mtime) = 0; virtual int get(string& entry, RGWMetadataObject **obj, optional_yield) = 0; - virtual int put(string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield, RGWMDLogSyncType type) = 0; + virtual int put(string& entry, + RGWMetadataObject *obj, + RGWObjVersionTracker& objv_tracker, + optional_yield, RGWMDLogSyncType type, + bool from_remote_zone) = 0; virtual int remove(string& entry, RGWObjVersionTracker& objv_tracker, optional_yield) = 0; virtual int mutate(const string& entry, @@ -106,7 +110,8 @@ protected: virtual int do_get(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject **obj, optional_yield y) = 0; virtual int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject *obj, - RGWObjVersionTracker& objv_tracker, optional_yield y, RGWMDLogSyncType type) = 0; + RGWObjVersionTracker& objv_tracker, optional_yield y, + RGWMDLogSyncType type, bool from_remote_zone) = 0; virtual int do_put_operate(Put *put_op); virtual int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, optional_yield y) = 0; @@ -132,6 +137,7 @@ public: RGWObjVersionTracker& objv_tracker; RGWMDLogSyncType apply_type; optional_yield y; + bool from_remote_zone{false}; int get(RGWMetadataObject **obj) { return handler->do_get(op, entry, obj, y); @@ -140,7 +146,7 @@ public: Put(RGWMetadataHandler_GenericMetaBE *_handler, RGWSI_MetaBackend_Handler::Op *_op, string& _entry, RGWMetadataObject *_obj, RGWObjVersionTracker& _objv_tracker, optional_yield _y, - RGWMDLogSyncType _type); + RGWMDLogSyncType _type, bool from_remote_zone); virtual ~Put() {} @@ -159,7 +165,7 @@ public: }; int get(string& entry, RGWMetadataObject **obj, optional_yield) override; - int put(string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield, RGWMDLogSyncType type) override; + int put(string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield, RGWMDLogSyncType type, bool from_remote_zone) override; int remove(string& entry, RGWObjVersionTracker& objv_tracker, optional_yield) override; int mutate(const string& entry, @@ -230,6 +236,7 @@ public: int get(string& metadata_key, Formatter *f, optional_yield y); int put(string& metadata_key, bufferlist& bl, optional_yield y, RGWMDLogSyncType sync_mode, + bool from_remote_zone, obj_version *existing_version = NULL); int remove(string& metadata_key, optional_yield y); @@ -267,7 +274,7 @@ public: RGWMetadataHandlerPut_SObj(RGWMetadataHandler_GenericMetaBE *handler, RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType type); + RGWMDLogSyncType type, bool from_remote_zone); ~RGWMetadataHandlerPut_SObj(); int put_pre() override; diff --git a/src/rgw/rgw_otp.cc b/src/rgw/rgw_otp.cc index 57a3183ce4b4..30d825cb3ca3 100644 --- a/src/rgw/rgw_otp.cc +++ b/src/rgw/rgw_otp.cc @@ -119,7 +119,7 @@ class RGWOTPMetadataHandler : public RGWOTPMetadataHandlerBase { int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType type) override { + RGWMDLogSyncType type, bool from_remote_zone) override { RGWOTPMetadataObject *obj = static_cast(_obj); RGWSI_OTP_BE_Ctx be_ctx(op->ctx()); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 65e07196f311..7cc9136757e7 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -2235,15 +2235,14 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket, info.owner = owner.user_id; info.zonegroup = zonegroup_id; info.placement_rule = selected_placement_rule; - info.layout.current_index.layout.type = rule_info.index_type; info.swift_ver_location = swift_ver_location; info.swift_versioning = (!swift_ver_location.empty()); + init_default_bucket_layout(cct, info, svc.zone->get_zone()); if (pmaster_num_shards) { + // TODO: remove this once sync doesn't require the same shard count info.layout.current_index.layout.normal.num_shards = *pmaster_num_shards; - } else { - info.layout.current_index.layout.normal.num_shards = bucket_index_max_shards; } - info.layout.current_index.layout.normal.hash_type = rgw::BucketHashType::Mod; + info.layout.current_index.layout.type = rule_info.index_type; info.requester_pays = false; if (real_clock::is_zero(creation_time)) { info.creation_time = ceph::real_clock::now(); diff --git a/src/rgw/rgw_sync.cc b/src/rgw/rgw_sync.cc index 77a7d885b669..66bec533ce64 100644 --- a/src/rgw/rgw_sync.cc +++ b/src/rgw/rgw_sync.cc @@ -1086,7 +1086,7 @@ class RGWAsyncMetaStoreEntry : public RGWAsyncRadosRequest { bufferlist bl; protected: int _send_request() override { - int ret = store->ctl()->meta.mgr->put(raw_key, bl, null_yield, RGWMDLogSyncType::APPLY_ALWAYS); + int ret = store->ctl()->meta.mgr->put(raw_key, bl, null_yield, RGWMDLogSyncType::APPLY_ALWAYS, true); if (ret < 0) { ldout(store->ctx(), 0) << "ERROR: can't store key: " << raw_key << " ret=" << ret << dendl; return ret; diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc index ad4f6b06127c..59ae19686d10 100644 --- a/src/rgw/rgw_user.cc +++ b/src/rgw/rgw_user.cc @@ -2587,7 +2587,7 @@ public: RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType type) override; + RGWMDLogSyncType type, bool from_remote_zone) override; int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, optional_yield y) override { @@ -2616,7 +2616,7 @@ public: RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType type) : RGWMetadataHandlerPut_SObj(_handler, op, entry, obj, objv_tracker, y, type), + RGWMDLogSyncType type, bool from_remote_zone) : RGWMetadataHandlerPut_SObj(_handler, op, entry, obj, objv_tracker, y, type, from_remote_zone), uhandler(_handler) { uobj = static_cast(obj); } @@ -2628,9 +2628,9 @@ int RGWUserMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op *op, string& en RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, optional_yield y, - RGWMDLogSyncType type) + RGWMDLogSyncType type, bool from_remote_zone) { - RGWMetadataHandlerPut_User put_op(this, op, entry, obj, objv_tracker, y, type); + RGWMetadataHandlerPut_User put_op(this, op, entry, obj, objv_tracker, y, type, from_remote_zone); return do_put_operate(&put_op); } -- 2.47.3