RGW:Multisite: Verify if the synced object is identical to source

Introduce an option 'rgw_copy_verify_object' which allows the object
copied from remote cluster through multisite sync is identical to the
source object. This is done by generating the MD5 checksum of the data
being copied and compared to the ETAG stored as part of the object's
attribute.

Signed-off-by: Prasad Krishnan <prasad.krishnan@flipkart.com>
(cherry picked from commit 23fb5d7b786f1034a02c991c6a648e455b5b9c30)

Conflicts:
	src/rgw/rgw_rados.cc

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index d4c6a8e908dd496fec8c39dfc142626548434745..774e88659ab179834ab9299a97750adff5b9b8e9 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -1476,6 +1476,7 @@ OPTION(rgw_curl_low_speed_limit, OPT_INT) // low speed limit for certain curl ca
 OPTION(rgw_curl_low_speed_time, OPT_INT) // low speed time for certain curl calls
 OPTION(rgw_copy_obj_progress, OPT_BOOL) // should dump progress during long copy operations?
 OPTION(rgw_copy_obj_progress_every_bytes, OPT_INT) // min bytes between copy progress output
+OPTION(rgw_copy_verify_object, OPT_BOOL) // verify if the copied object is identical to source
 OPTION(rgw_obj_tombstone_cache_size, OPT_INT) // how many objects in tombstone cache, which is used in multi-zone sync to keep
                                                     // track of removed objects' mtime
 

diff --git a/src/common/options.cc b/src/common/options.cc
index b031cf0ada39155c4b82e3ed1b08545b7ab0ce69..4f400823708295f4ac34d41d1902803b8e76dcb5 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6643,6 +6643,13 @@ std::vector<Option> get_rgw_options() {
     .set_default(1_M)
     .set_description("Send copy-object progress info after these many bytes"),
 
+    Option("rgw_copy_verify_object", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(false)
+    .set_description("Verify if the object copied is identical to its source")
+    .set_long_description(
+        "If true, this option computes the MD5 checksum of the data which is written at the"
+       "destination and checks if it is identical to the ETAG stored in the source."),
+
     Option("rgw_obj_tombstone_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(1000)
     .set_description("Max number of entries to keep in tombstone cache")

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index cda2b1ed072280723483ccbe19c2b33d9ed26295..fc0d732969425b3104201ba4b4ea9e2aae4a811c 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -3924,7 +3924,7 @@ class RGWRadosPutObj : public RGWHTTPStreamRWRequest::ReceiveCB
   rgw::putobj::ObjectProcessor *processor;
   void (*progress_cb)(off_t, void *);
   void *progress_data;
-  bufferlist extra_data_bl;
+  bufferlist extra_data_bl, full_obj;
   uint64_t extra_data_left{0};
   bool need_to_process_attrs{true};
   uint64_t data_len{0};
@@ -3932,6 +3932,7 @@ class RGWRadosPutObj : public RGWHTTPStreamRWRequest::ReceiveCB
   uint64_t ofs{0};
   uint64_t lofs{0}; /* logical ofs */
   std::function<int(const map<string, bufferlist>&)> attrs_handler;
+  string calculated_etag;
 public:
   RGWRadosPutObj(CephContext* cct,
                  CompressorRef& plugin,
@@ -4034,6 +4035,8 @@ public:
 
     const uint64_t lofs = data_len;
     data_len += size;
+    if (cct->_conf->rgw_copy_verify_object)
+      full_obj.append(bl);
 
     return filter->process(std::move(bl), lofs);
   }
@@ -4054,6 +4057,18 @@ public:
   uint64_t get_data_len() {
     return data_len;
   }
+
+  string get_calculated_etag() {
+    MD5 hash;
+    unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE];
+    char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
+
+    hash.Update((const unsigned char *)full_obj.c_str(), data_len);
+    hash.Final(m);
+    buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5);
+    calculated_etag = calc_md5;
+    return calculated_etag;
+  }
 };
 
 /*
@@ -4545,6 +4560,16 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
     if (ret < 0) {
       goto set_err_state;
     }
+
+    if (cct->_conf->rgw_copy_verify_object) {
+      if (cb.get_calculated_etag().compare(etag)) {
+        ret = -EIO;
+        ldout(cct, 0) << "ERROR: source and destination objects don't match. Expected etag:"
+          << etag << " Computed etag:" << cb.get_calculated_etag() << dendl;
+        goto set_err_state;
+     }
+   }
+
     if (copy_if_newer && canceled) {
       ldout(cct, 20) << "raced with another write of obj: " << dest_obj << dendl;
       obj_ctx.invalidate(dest_obj); /* object was overwritten */