rgw: write meta of a MP part to a correct pool

When uploading a multipart part to a non-default storage class,
original logic writes the object data to the correct pool
for its storage class.
But when completing the upload, there's another write op
(with a bunch of setxattrs) to the head object targeting
the standard pool (because non-multipart uploads
write their head objects to the default pool).
As a result of this setxattrs op, a redundant zero-sized object is
created in a wrong pool. Multipart complete, multipart abort and
GC would never try to it clean up.

The fix is to make RGWRados::Object::Write::_do_write_meta() learn the
difference between multipart and non-multipart head objects and pick the
correct placement rule for RGWRados::get_obj_head_ref().

Fixes: http://tracker.ceph.com/issues/49128
Signed-off-by: Jeegn Chen <jeegnchen@tencent.com>

diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc
index 687583e3303c5fc604b6a2a20c62bf87c0b529ed..270ccf0960c3a3fb26f605c50838088036ba6e39 100644 (file)
--- a/src/rgw/rgw_putobj_processor.cc
+++ b/src/rgw/rgw_putobj_processor.cc
@@ -328,6 +328,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
   obj_op->params.zones_trace = zones_trace;
   obj_op->params.modify_tail = true;
   obj_op->params.attrs = &attrs;
+  obj_op->params.pmeta_placement_rule = &tail_placement_rule;
   r = obj_op->prepare(y);
   if (r < 0) {
     return r;

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 7acff692131e24d4f15404857b30acea1a6ed71f..65c9745244250cc06425724abcc713ce1b0240de 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -2352,12 +2352,15 @@ int RGWRados::get_obj_head_ioctx(const DoutPrefixProvider *dpp, const RGWBucketI
   return 0;
 }
 
-int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_rados_ref *ref)
+int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp,
+                               const rgw_placement_rule& target_placement_rule,
+                               const rgw_obj& obj,
+                               rgw_rados_ref *ref)
 {
   get_obj_bucket_and_oid_loc(obj, ref->obj.oid, ref->obj.loc);
 
   rgw_pool pool;
-  if (!get_obj_data_pool(bucket_info.placement_rule, obj, &pool)) {
+  if (!get_obj_data_pool(target_placement_rule, obj, &pool)) {
     ldpp_dout(dpp, 0) << "ERROR: cannot get data pool for obj=" << obj << ", probably misconfiguration" << dendl;
     return -EIO;
   }
@@ -2376,6 +2379,14 @@ int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInf
   return 0;
 }
 
+int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp,
+                               const RGWBucketInfo& bucket_info,
+                               const rgw_obj& obj,
+                               rgw_rados_ref *ref)
+{
+  return get_obj_head_ref(dpp, bucket_info.placement_rule, obj, ref);
+}
+
 int RGWRados::get_raw_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref)
 {
   ref->obj = obj;
@@ -3026,7 +3037,7 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
   }
 
   rgw_rados_ref ref;
-  r = store->get_obj_head_ref(dpp, target->get_bucket_info(), obj, &ref);
+  r = store->get_obj_head_ref(dpp, target->get_meta_placement_rule(), obj, &ref);
   if (r < 0)
     return r;
 

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 6b695899f56c9e17aee40cbd697ddfdf4ddc3134..f73166d35afa3669cc7476c78561b6698fae1a28 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -434,6 +434,7 @@ class RGWRados
   // This field represents the number of bucket index object shards
   uint32_t bucket_index_max_shards;
 
+  int get_obj_head_ref(const DoutPrefixProvider *dpp, const rgw_placement_rule& target_placement_rule, const rgw_obj& obj, rgw_rados_ref *ref);
   int get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_rados_ref *ref);
   int get_system_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref);
   uint64_t max_bucket_id;
@@ -690,6 +691,8 @@ public:
 
     bool bs_initialized;
 
+    const rgw_placement_rule *pmeta_placement_rule;
+
   protected:
     int get_state(const DoutPrefixProvider *dpp, RGWObjState **pstate, bool follow_olh, optional_yield y, bool assume_noent = false);
     void invalidate_state();
@@ -702,7 +705,8 @@ public:
     Object(RGWRados *_store, const RGWBucketInfo& _bucket_info, RGWObjectCtx& _ctx, const rgw_obj& _obj) : store(_store), bucket_info(_bucket_info),
                                                                                                ctx(_ctx), obj(_obj), bs(store),
                                                                                                state(NULL), versioning_disabled(false),
-                                                                                               bs_initialized(false) {}
+                                                                                               bs_initialized(false),
+                                                                                               pmeta_placement_rule(nullptr) {}
 
     RGWRados *get_store() { return store; }
     rgw_obj& get_obj() { return obj; }
@@ -731,6 +735,14 @@ public:
       return (!versioning_disabled && bucket_info.versioning_enabled());
     }
 
+    void set_meta_placement_rule(const rgw_placement_rule *p) {
+        pmeta_placement_rule = p;
+    }
+
+    const rgw_placement_rule& get_meta_placement_rule() {
+        return pmeta_placement_rule ? *pmeta_placement_rule : bucket_info.placement_rule;
+    }
+
     struct Read {
       RGWRados::Object *source;
 

diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 5686ff5b337d03d038a66959f97eb310c9918030..59d934c3b0c16979d2df9b1a571665dac2cf73d6 100644 (file)
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -569,6 +569,10 @@ class Object {
        bool completeMultipart{false};
        bool appendable{false};
        Attrs* attrs{nullptr};
+       // In MultipartObjectProcessor::complete, we need this parameter
+       // to tell the exact placement rule since it may be different from
+       // bucket.placement_rule when Storage Class is specified explicitly
+       const rgw_placement_rule *pmeta_placement_rule{nullptr};
       } params;
 
       virtual ~WriteOp() = default;

diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc
index 2c6f66121d9e5af5ee81f91c1dcb950e890cc888..7624e329e98b2141e6403f364fa13bf594e9c178 100644 (file)
--- a/src/rgw/rgw_sal_rados.cc
+++ b/src/rgw/rgw_sal_rados.cc
@@ -1719,6 +1719,7 @@ RadosObject::RadosWriteOp::RadosWriteOp(RadosObject* _source, RGWObjectCtx* _rct
 int RadosObject::RadosWriteOp::prepare(optional_yield y)
 {
   op_target.set_versioning_disabled(params.versioning_disabled);
+  op_target.set_meta_placement_rule(params.pmeta_placement_rule);
   parent_op.meta.mtime = params.mtime;
   parent_op.meta.rmattrs = params.rmattrs;
   parent_op.meta.data = params.data;