rgw: write meta of a MP part to a correct pool

When uploading a multipart part to a non-default storage class,
original logic writes the object data to the correct pool
for its storage class.
But when completing the upload, there's another write op
(with a bunch of setxattrs) to the head object targeting
the standard pool (because non-multipart uploads
write their head objects to the default pool).
As a result of this setxattrs op, a redundant zero-sized object is
created in a wrong pool. Multipart complete, multipart abort and
GC would never try to it clean up.

The fix is to make RGWRados::Object::Write::_do_write_meta() learn the
difference between multipart and non-multipart head objects and pick the
correct placement rule for RGWRados::get_obj_head_ref().

Fixes: http://tracker.ceph.com/issues/49128
Signed-off-by: Jeegn Chen <jeegnchen@tencent.com>
(cherry picked from commit 7647823bd42ce8228c3622a91e87680ee22df710)

Conflicts:
	src/rgw/rgw_sal.h

Cherry-pick notes:
- RGWAttrs renamed to Attrs after pacific

diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc
index 6be1cecda19036b7c8c6b3121dbf97aa857b9397..68730173edfb68986a9b9180591f219b9d7e8dbe 100644 (file)
--- a/src/rgw/rgw_putobj_processor.cc
+++ b/src/rgw/rgw_putobj_processor.cc
@@ -435,6 +435,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
   obj_op->params.zones_trace = zones_trace;
   obj_op->params.modify_tail = true;
   obj_op->params.attrs = &attrs;
+  obj_op->params.pmeta_placement_rule = &tail_placement_rule;
   r = obj_op->prepare(y);
   if (r < 0) {
     return r;

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index b9f3e08f59e06c7bce79d5beb9973193fc444cff..3af32427d07ca48a142c93818c4e3493ce2ecf59 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -2351,12 +2351,15 @@ int RGWRados::get_obj_head_ioctx(const DoutPrefixProvider *dpp, const RGWBucketI
   return 0;
 }
 
-int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_rados_ref *ref)
+int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp,
+                               const rgw_placement_rule& target_placement_rule,
+                               const rgw_obj& obj,
+                               rgw_rados_ref *ref)
 {
   get_obj_bucket_and_oid_loc(obj, ref->obj.oid, ref->obj.loc);
 
   rgw_pool pool;
-  if (!get_obj_data_pool(bucket_info.placement_rule, obj, &pool)) {
+  if (!get_obj_data_pool(target_placement_rule, obj, &pool)) {
     ldpp_dout(dpp, 0) << "ERROR: cannot get data pool for obj=" << obj << ", probably misconfiguration" << dendl;
     return -EIO;
   }
@@ -2375,6 +2378,14 @@ int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInf
   return 0;
 }
 
+int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp,
+                               const RGWBucketInfo& bucket_info,
+                               const rgw_obj& obj,
+                               rgw_rados_ref *ref)
+{
+  return get_obj_head_ref(dpp, bucket_info.placement_rule, obj, ref);
+}
+
 int RGWRados::get_raw_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref)
 {
   ref->obj = obj;
@@ -3025,7 +3036,7 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
   }
 
   rgw_rados_ref ref;
-  r = store->get_obj_head_ref(dpp, target->get_bucket_info(), obj, &ref);
+  r = store->get_obj_head_ref(dpp, target->get_meta_placement_rule(), obj, &ref);
   if (r < 0)
     return r;
 

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index b1a51ba335641ebacf48af856ba3bc10344cfea4..827419a0ab9e0904a9b844c2ef55a0ac61f1e52b 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -461,6 +461,7 @@ class RGWRados
   // This field represents the number of bucket index object shards
   uint32_t bucket_index_max_shards;
 
+  int get_obj_head_ref(const DoutPrefixProvider *dpp, const rgw_placement_rule& target_placement_rule, const rgw_obj& obj, rgw_rados_ref *ref);
   int get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_rados_ref *ref);
   int get_system_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref);
   uint64_t max_bucket_id;
@@ -717,6 +718,8 @@ public:
 
     bool bs_initialized;
 
+    const rgw_placement_rule *pmeta_placement_rule;
+
   protected:
     int get_state(const DoutPrefixProvider *dpp, RGWObjState **pstate, bool follow_olh, optional_yield y, bool assume_noent = false);
     void invalidate_state();
@@ -729,7 +732,8 @@ public:
     Object(RGWRados *_store, const RGWBucketInfo& _bucket_info, RGWObjectCtx& _ctx, const rgw_obj& _obj) : store(_store), bucket_info(_bucket_info),
                                                                                                ctx(_ctx), obj(_obj), bs(store),
                                                                                                state(NULL), versioning_disabled(false),
-                                                                                               bs_initialized(false) {}
+                                                                                               bs_initialized(false),
+                                                                                               pmeta_placement_rule(nullptr) {}
 
     RGWRados *get_store() { return store; }
     rgw_obj& get_obj() { return obj; }
@@ -758,6 +762,14 @@ public:
       return (!versioning_disabled && bucket_info.versioning_enabled());
     }
 
+    void set_meta_placement_rule(const rgw_placement_rule *p) {
+        pmeta_placement_rule = p;
+    }
+
+    const rgw_placement_rule& get_meta_placement_rule() {
+        return pmeta_placement_rule ? *pmeta_placement_rule : bucket_info.placement_rule;
+    }
+
     struct Read {
       RGWRados::Object *source;
 

diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 41da8bc9d854e7cce2888286d64d2326c3133cb3..9a6a7c168d18bf01a62baa72152b0fd5b87a6100 100644 (file)
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -409,6 +409,10 @@ class RGWObject {
        bool completeMultipart{false};
        bool appendable{false};
        RGWAttrs* attrs{nullptr};
+       // In MultipartObjectProcessor::complete, we need this parameter
+       // to tell the exact placement rule since it may be different from
+       // bucket.placement_rule when Storage Class is specified explicitly
+       const rgw_placement_rule *pmeta_placement_rule{nullptr};
       } params;
 
       virtual ~WriteOp() = default;

diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc
index 28f1cef9ffefd83afbef14739677cc273d217270..b5ef71fbc0d3a41a51c6a4ca0a1284bc89767b8c 100644 (file)
--- a/src/rgw/rgw_sal_rados.cc
+++ b/src/rgw/rgw_sal_rados.cc
@@ -809,6 +809,7 @@ RGWRadosObject::RadosWriteOp::RadosWriteOp(RGWRadosObject* _source, RGWObjectCtx
 int RGWRadosObject::RadosWriteOp::prepare(optional_yield y)
 {
   op_target.set_versioning_disabled(params.versioning_disabled);
+  op_target.set_meta_placement_rule(params.pmeta_placement_rule);
   parent_op.meta.mtime = params.mtime;
   parent_op.meta.rmattrs = params.rmattrs;
   parent_op.meta.data = params.data;