rgw: rgw_sync_obj_etag_verify accounts for compressed multipart uploads

the etag verifier for multipart uploads uses the manifest to get the
logical offsets for each part. but when compression is enabled, those
are offsets into the compressed data. use the source object's compression
info to translate those compressed part offsets back to their original
offsets

Fixes: https://tracker.ceph.com/issues/45992
Signed-off-by: Casey Bodley <cbodley@redhat.com>
(cherry picked from commit 51f79fad8398d35e07f01fa45704124e16fadeec)

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 62cb05837e71d71dc604996276fb676c05e723b3..ae3bbad87eb428d7ccb0552587ce3daa6d85fd79 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -3920,6 +3920,7 @@ class RGWRadosPutObj : public RGWHTTPStreamRWRequest::ReceiveCB
   rgw_obj obj;
   rgw::putobj::DataProcessor *filter;
   boost::optional<RGWPutObj_Compress>& compressor;
+  bool try_etag_verify;
   boost::optional<RGWPutObj_ETagVerifier_Atomic> etag_verifier_atomic;
   boost::optional<RGWPutObj_ETagVerifier_MPU> etag_verifier_mpu;
   boost::optional<rgw::putobj::ChunkProcessor> buffering;
@@ -3928,6 +3929,7 @@ class RGWRadosPutObj : public RGWHTTPStreamRWRequest::ReceiveCB
   void (*progress_cb)(off_t, void *);
   void *progress_data;
   bufferlist extra_data_bl, manifest_bl;
+  std::optional<RGWCompressionInfo> compression_info;
   uint64_t extra_data_left{0};
   bool need_to_process_attrs{true};
   SourceObjType obj_type{OBJ_TYPE_UNINIT};
@@ -3947,6 +3949,7 @@ public:
                        cct(cct),
                        filter(p),
                        compressor(compressor),
+                       try_etag_verify(cct->_conf->rgw_sync_obj_etag_verify),
                        plugin(plugin),
                        processor(p),
                        progress_cb(_progress_cb),
@@ -3963,9 +3966,28 @@ public:
 
       JSONDecoder::decode_json("attrs", src_attrs, &jp);
 
-      src_attrs.erase(RGW_ATTR_COMPRESSION);
+      auto iter = src_attrs.find(RGW_ATTR_COMPRESSION);
+      if (iter != src_attrs.end()) {
+        const bufferlist bl = std::move(iter->second);
+        src_attrs.erase(iter); // don't preserve source compression info
+
+        if (try_etag_verify) {
+          // if we're trying to verify etags, we need to convert compressed
+          // ranges in the manifest back into logical multipart part offsets
+          RGWCompressionInfo info;
+          bool compressed = false;
+          int r = rgw_compression_info_from_attr(bl, compressed, info);
+          if (r < 0) {
+            ldout(cct, 4) << "failed to decode compression info, "
+                "disabling etag verification" << dendl;
+            try_etag_verify = false;
+          } else if (compressed) {
+            compression_info = std::move(info);
+          }
+        }
+      }
       /* We need the manifest to recompute the ETag for verification */
-      auto iter = src_attrs.find(RGW_ATTR_MANIFEST);
+      iter = src_attrs.find(RGW_ATTR_MANIFEST);
       if (iter != src_attrs.end()) {
         manifest_bl = std::move(iter->second);
         src_attrs.erase(iter);
@@ -4002,8 +4024,7 @@ public:
      * requested. We can enable simultaneous support once we have a mechanism
      * to know the sequence in which the filters must be applied.
      */
-    if (cct->_conf->rgw_sync_obj_etag_verify &&
-        src_attrs.find(RGW_ATTR_CRYPT_MODE) == src_attrs.end()) {
+    if (try_etag_verify && src_attrs.find(RGW_ATTR_CRYPT_MODE) == src_attrs.end()) {
 
       RGWObjManifest manifest;
 
@@ -4044,9 +4065,36 @@ public:
           part_ofs.push_back(cur_part_ofs);
         }
 
-        obj_type = OBJ_TYPE_MPU;
-        etag_verifier_mpu = boost::in_place(cct, std::move(part_ofs), filter);
-        filter = &*etag_verifier_mpu;
+        if (compression_info) {
+          // if the source object was compressed, the manifest is storing
+          // compressed part offsets. transform the compressed offsets back to
+          // their original offsets by finding the first block of each part
+          const auto& blocks = compression_info->blocks;
+          auto block = blocks.begin();
+          for (auto& ofs : part_ofs) {
+            // find the compression_block with new_ofs == ofs
+            constexpr auto less = [] (const compression_block& block, uint64_t ofs) {
+              return block.new_ofs < ofs;
+            };
+            block = std::lower_bound(block, blocks.end(), ofs, less);
+            if (block == blocks.end() || block->new_ofs != ofs) {
+              ldout(cct, 4) << "no match for compressed offset " << ofs
+                  << ", disabling etag verification" << dendl;
+              part_ofs.clear();
+              break;
+            }
+            ofs = block->old_ofs;
+            ldout(cct, 20) << "MPU Part uncompressed offset:" << ofs << dendl;
+          }
+        }
+
+        if (part_ofs.empty()) {
+          try_etag_verify = false;
+        } else {
+          obj_type = OBJ_TYPE_MPU;
+          etag_verifier_mpu = boost::in_place(cct, std::move(part_ofs), filter);
+          filter = &*etag_verifier_mpu;
+        }
       }
     }