From: Jeegn Chen Date: Mon, 8 Mar 2021 14:27:44 +0000 (+0800) Subject: rgw: write meta of a MP part to a correct pool X-Git-Tag: v16.2.8~107^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F45428%2Fhead;p=ceph.git rgw: write meta of a MP part to a correct pool When uploading a multipart part to a non-default storage class, original logic writes the object data to the correct pool for its storage class. But when completing the upload, there's another write op (with a bunch of setxattrs) to the head object targeting the standard pool (because non-multipart uploads write their head objects to the default pool). As a result of this setxattrs op, a redundant zero-sized object is created in a wrong pool. Multipart complete, multipart abort and GC would never try to it clean up. The fix is to make RGWRados::Object::Write::_do_write_meta() learn the difference between multipart and non-multipart head objects and pick the correct placement rule for RGWRados::get_obj_head_ref(). Fixes: http://tracker.ceph.com/issues/49128 Signed-off-by: Jeegn Chen (cherry picked from commit 7647823bd42ce8228c3622a91e87680ee22df710) Conflicts: src/rgw/rgw_sal.h Cherry-pick notes: - RGWAttrs renamed to Attrs after pacific --- diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc index 6be1cecda190..68730173edfb 100644 --- a/src/rgw/rgw_putobj_processor.cc +++ b/src/rgw/rgw_putobj_processor.cc @@ -435,6 +435,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size, obj_op->params.zones_trace = zones_trace; obj_op->params.modify_tail = true; obj_op->params.attrs = &attrs; + obj_op->params.pmeta_placement_rule = &tail_placement_rule; r = obj_op->prepare(y); if (r < 0) { return r; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index b9f3e08f59e0..3af32427d07c 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -2351,12 +2351,15 @@ int RGWRados::get_obj_head_ioctx(const DoutPrefixProvider *dpp, const RGWBucketI return 0; } -int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_rados_ref *ref) +int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp, + const rgw_placement_rule& target_placement_rule, + const rgw_obj& obj, + rgw_rados_ref *ref) { get_obj_bucket_and_oid_loc(obj, ref->obj.oid, ref->obj.loc); rgw_pool pool; - if (!get_obj_data_pool(bucket_info.placement_rule, obj, &pool)) { + if (!get_obj_data_pool(target_placement_rule, obj, &pool)) { ldpp_dout(dpp, 0) << "ERROR: cannot get data pool for obj=" << obj << ", probably misconfiguration" << dendl; return -EIO; } @@ -2375,6 +2378,14 @@ int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInf return 0; } +int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + const rgw_obj& obj, + rgw_rados_ref *ref) +{ + return get_obj_head_ref(dpp, bucket_info.placement_rule, obj, ref); +} + int RGWRados::get_raw_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref) { ref->obj = obj; @@ -3025,7 +3036,7 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp, } rgw_rados_ref ref; - r = store->get_obj_head_ref(dpp, target->get_bucket_info(), obj, &ref); + r = store->get_obj_head_ref(dpp, target->get_meta_placement_rule(), obj, &ref); if (r < 0) return r; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index b1a51ba33564..827419a0ab9e 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -461,6 +461,7 @@ class RGWRados // This field represents the number of bucket index object shards uint32_t bucket_index_max_shards; + int get_obj_head_ref(const DoutPrefixProvider *dpp, const rgw_placement_rule& target_placement_rule, const rgw_obj& obj, rgw_rados_ref *ref); int get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_rados_ref *ref); int get_system_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref); uint64_t max_bucket_id; @@ -717,6 +718,8 @@ public: bool bs_initialized; + const rgw_placement_rule *pmeta_placement_rule; + protected: int get_state(const DoutPrefixProvider *dpp, RGWObjState **pstate, bool follow_olh, optional_yield y, bool assume_noent = false); void invalidate_state(); @@ -729,7 +732,8 @@ public: Object(RGWRados *_store, const RGWBucketInfo& _bucket_info, RGWObjectCtx& _ctx, const rgw_obj& _obj) : store(_store), bucket_info(_bucket_info), ctx(_ctx), obj(_obj), bs(store), state(NULL), versioning_disabled(false), - bs_initialized(false) {} + bs_initialized(false), + pmeta_placement_rule(nullptr) {} RGWRados *get_store() { return store; } rgw_obj& get_obj() { return obj; } @@ -758,6 +762,14 @@ public: return (!versioning_disabled && bucket_info.versioning_enabled()); } + void set_meta_placement_rule(const rgw_placement_rule *p) { + pmeta_placement_rule = p; + } + + const rgw_placement_rule& get_meta_placement_rule() { + return pmeta_placement_rule ? *pmeta_placement_rule : bucket_info.placement_rule; + } + struct Read { RGWRados::Object *source; diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 41da8bc9d854..9a6a7c168d18 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -409,6 +409,10 @@ class RGWObject { bool completeMultipart{false}; bool appendable{false}; RGWAttrs* attrs{nullptr}; + // In MultipartObjectProcessor::complete, we need this parameter + // to tell the exact placement rule since it may be different from + // bucket.placement_rule when Storage Class is specified explicitly + const rgw_placement_rule *pmeta_placement_rule{nullptr}; } params; virtual ~WriteOp() = default; diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc index 28f1cef9ffef..b5ef71fbc0d3 100644 --- a/src/rgw/rgw_sal_rados.cc +++ b/src/rgw/rgw_sal_rados.cc @@ -809,6 +809,7 @@ RGWRadosObject::RadosWriteOp::RadosWriteOp(RGWRadosObject* _source, RGWObjectCtx int RGWRadosObject::RadosWriteOp::prepare(optional_yield y) { op_target.set_versioning_disabled(params.versioning_disabled); + op_target.set_meta_placement_rule(params.pmeta_placement_rule); parent_op.meta.mtime = params.mtime; parent_op.meta.rmattrs = params.rmattrs; parent_op.meta.data = params.data;