RGW:Multisite: Verify if the synced object is identical to source

Introduce an option 'rgw_copy_verify_object' which allows the object
copied from remote cluster through multisite sync is identical to the
source object. This is done by generating the MD5 checksum of the data
being copied and compared to the ETAG stored as part of the object's
attribute.

Signed-off-by: Prasad Krishnan <prasad.krishnan@flipkart.com>
(cherry picked from commit 23fb5d7b786f1034a02c991c6a648e455b5b9c30)

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index ca8f252b7fbe4d9059632e567aa3188abe284c0a..1084eae01b101b47b8778864ab67ca6e1634e3de 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -1446,6 +1446,7 @@ OPTION(rgw_curl_low_speed_limit, OPT_INT) // low speed limit for certain curl ca
 OPTION(rgw_curl_low_speed_time, OPT_INT) // low speed time for certain curl calls
 OPTION(rgw_copy_obj_progress, OPT_BOOL) // should dump progress during long copy operations?
 OPTION(rgw_copy_obj_progress_every_bytes, OPT_INT) // min bytes between copy progress output
+OPTION(rgw_copy_verify_object, OPT_BOOL) // verify if the copied object is identical to source
 OPTION(rgw_obj_tombstone_cache_size, OPT_INT) // how many objects in tombstone cache, which is used in multi-zone sync to keep
                                                     // track of removed objects' mtime
 

diff --git a/src/common/options.cc b/src/common/options.cc
index 73a483fa1cacf4057fe6595de6baacc756d83165..3540c239d9a97698334e59fdce251a68657d4ce2 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6390,6 +6390,13 @@ std::vector<Option> get_rgw_options() {
     .set_default(1_M)
     .set_description("Send copy-object progress info after these many bytes"),
 
+    Option("rgw_copy_verify_object", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(false)
+    .set_description("Verify if the object copied is identical to its source")
+    .set_long_description(
+        "If true, this option computes the MD5 checksum of the data which is written at the"
+       "destination and checks if it is identical to the ETAG stored in the source."),
+
     Option("rgw_obj_tombstone_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(1000)
     .set_description("Max number of entries to keep in tombstone cache")

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 2d74d5af48383e6c04a2f3d1bf4c76f674c6685f..9784971219107073b5307a69a3c8184312f3e731 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -3261,7 +3261,7 @@ class RGWRadosPutObj : public RGWHTTPStreamRWRequest::ReceiveCB
   rgw::putobj::ObjectProcessor *processor;
   void (*progress_cb)(off_t, void *);
   void *progress_data;
-  bufferlist extra_data_bl;
+  bufferlist extra_data_bl, full_obj;
   uint64_t extra_data_left{0};
   bool need_to_process_attrs{true};
   uint64_t data_len{0};
@@ -3269,6 +3269,7 @@ class RGWRadosPutObj : public RGWHTTPStreamRWRequest::ReceiveCB
   uint64_t ofs{0};
   uint64_t lofs{0}; /* logical ofs */
   std::function<int(map<string, bufferlist>&)> attrs_handler;
+  string calculated_etag;
 public:
   RGWRadosPutObj(CephContext* cct,
                  CompressorRef& plugin,
@@ -3371,6 +3372,8 @@ public:
 
     const uint64_t lofs = data_len;
     data_len += size;
+    if (cct->_conf->rgw_copy_verify_object)
+      full_obj.append(bl);
 
     return filter->process(std::move(bl), lofs);
   }
@@ -3391,6 +3394,18 @@ public:
   uint64_t get_data_len() {
     return data_len;
   }
+
+  string get_calculated_etag() {
+    MD5 hash;
+    unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE];
+    char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
+
+    hash.Update((const unsigned char *)full_obj.c_str(), data_len);
+    hash.Final(m);
+    buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5);
+    calculated_etag = calc_md5;
+    return calculated_etag;
+  }
 };
 
 /*
@@ -3953,6 +3968,16 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
     if (ret < 0) {
       goto set_err_state;
     }
+
+    if (cct->_conf->rgw_copy_verify_object) {
+      if (cb.get_calculated_etag().compare(etag)) {
+        ret = -EIO;
+        ldout(cct, 0) << "ERROR: source and destination objects don't match. Expected etag:"
+          << etag << " Computed etag:" << cb.get_calculated_etag() << dendl;
+        goto set_err_state;
+     }
+   }
+
     if (copy_if_newer && canceled) {
       ldout(cct, 20) << "raced with another write of obj: " << dest_obj << dendl;
       obj_ctx.invalidate(dest_obj); /* object was overwritten */