rbd-mirror: Add sparse read for sync image

Currently, the image sync do full read, and we shall add sparse read
to let the sync more efficiently.

Feature: http://tracker.ceph.com/issues/16780
Signed-off-by: tianqing <tianqing@unitedstack.com>

diff --git a/src/test/librados_test_stub/MockTestMemIoCtxImpl.h b/src/test/librados_test_stub/MockTestMemIoCtxImpl.h
index b15b241db625cdd9fc6e3189a9001a5fc377d5cb..3547e453f2119aaca6579a5aba3e22573e1b6882 100644 (file)
--- a/src/test/librados_test_stub/MockTestMemIoCtxImpl.h
+++ b/src/test/librados_test_stub/MockTestMemIoCtxImpl.h
@@ -79,6 +79,16 @@ public:
     return TestMemIoCtxImpl::notify(o, bl, timeout_ms, pbl);
   }
 
+  MOCK_METHOD5(sparse_read, int(const std::string& oid,
+                               uint64_t off,
+                               size_t len,
+                               std::map<uint64_t, uint64_t> *m,
+                               bufferlist *bl));
+  int do_sparse_read(const std::string& oid, uint64_t off, size_t len,
+                     std::map<uint64_t, uint64_t> *m, bufferlist *bl){
+     return TestMemIoCtxImpl::sparse_read(oid, off, len, m, bl);
+  }
+
   MOCK_METHOD4(read, int(const std::string& oid,
                          size_t len,
                          uint64_t off,
@@ -142,6 +152,7 @@ public:
     ON_CALL(*this, list_watchers(_, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_list_watchers));
     ON_CALL(*this, notify(_, _, _, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_notify));
     ON_CALL(*this, read(_, _, _, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_read));
+    ON_CALL(*this, sparse_read(_, _, _, _, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_sparse_read));
     ON_CALL(*this, remove(_, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_remove));
     ON_CALL(*this, selfmanaged_snap_create(_)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_selfmanaged_snap_create));
     ON_CALL(*this, selfmanaged_snap_remove(_)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_selfmanaged_snap_remove));

diff --git a/src/test/rbd_mirror/image_sync/test_mock_ObjectCopyRequest.cc b/src/test/rbd_mirror/image_sync/test_mock_ObjectCopyRequest.cc
index b018f16c21b3045be3b8f2c5677a5d846cfc9606..52d879cdc29da71367f41ab83607464777c49046 100644 (file)
--- a/src/test/rbd_mirror/image_sync/test_mock_ObjectCopyRequest.cc
+++ b/src/test/rbd_mirror/image_sync/test_mock_ObjectCopyRequest.cc
@@ -4,6 +4,7 @@
 #include "test/rbd_mirror/test_mock_fixture.h"
 #include "include/interval_set.h"
 #include "include/rbd/librbd.hpp"
+#include "include/rbd/object_map_types.h"
 #include "librbd/AioImageRequestWQ.h"
 #include "librbd/ImageCtx.h"
 #include "librbd/ImageState.h"
@@ -108,9 +109,10 @@ public:
                                      0, on_finish);
   }
 
-  void expect_read(librados::MockTestMemIoCtxImpl &mock_io_ctx, uint64_t offset,
-                   uint64_t length, int r) {
-    auto &expect = EXPECT_CALL(mock_io_ctx, read(_, length, offset, _));
+  void expect_sparse_read(librados::MockTestMemIoCtxImpl &mock_io_ctx, uint64_t offset,
+                          uint64_t length, int r) {
+
+    auto &expect = EXPECT_CALL(mock_io_ctx, sparse_read(_, offset, length, _, _));
     if (r < 0) {
       expect.WillOnce(Return(r));
     } else {
@@ -118,10 +120,10 @@ public:
     }
   }
 
-  void expect_read(librados::MockTestMemIoCtxImpl &mock_io_ctx,
+  void expect_sparse_read(librados::MockTestMemIoCtxImpl &mock_io_ctx,
                    const interval_set<uint64_t> &extents, int r) {
     for (auto extent : extents) {
-      expect_read(mock_io_ctx, extent.first, extent.second, r);
+      expect_sparse_read(mock_io_ctx, extent.first, extent.second, r);
       if (r < 0) {
         break;
       }
@@ -212,6 +214,7 @@ public:
     }
     m_snap_map[remote_snap_id] = local_snap_ids;
     m_local_snap_ids.push_back(local_snap_id);
+
     return 0;
   }
 
@@ -348,7 +351,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, Write) {
 
   InSequence seq;
   expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0);
-  expect_read(mock_remote_io_ctx, 0, one.range_end(), 0);
+  expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0);
   expect_write(mock_local_io_ctx, 0, one.range_end(), 0);
   expect_update_object_map(mock_local_image_ctx, mock_object_map,
                            m_local_snap_ids[0], OBJECT_EXISTS, 0);
@@ -381,7 +384,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, ReadError) {
 
   InSequence seq;
   expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0);
-  expect_read(mock_remote_io_ctx, 0, one.range_end(), -EINVAL);
+  expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(),  -EINVAL);
 
   request->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
@@ -412,7 +415,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, WriteError) {
 
   InSequence seq;
   expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0);
-  expect_read(mock_remote_io_ctx, 0, one.range_end(), 0);
+  expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0);
   expect_write(mock_local_io_ctx, 0, one.range_end(), -EINVAL);
 
   request->send();
@@ -455,9 +458,9 @@ TEST_F(TestMockImageSyncObjectCopyRequest, WriteSnaps) {
 
   InSequence seq;
   expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0);
-  expect_read(mock_remote_io_ctx, 0, one.range_end(), 0);
+  expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0);
   expect_write(mock_local_io_ctx, 0, one.range_end(), 0);
-  expect_read(mock_remote_io_ctx, two, 0);
+  expect_sparse_read(mock_remote_io_ctx, two, 0);
   expect_write(mock_local_io_ctx, two, 0);
   expect_update_object_map(mock_local_image_ctx, mock_object_map,
                            m_local_snap_ids[0], OBJECT_EXISTS, 0);
@@ -504,7 +507,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, Trim) {
 
   InSequence seq;
   expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0);
-  expect_read(mock_remote_io_ctx, 0, one.range_end(), 0);
+  expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0);
   expect_write(mock_local_io_ctx, 0, one.range_end(), 0);
   expect_truncate(mock_local_io_ctx, trim_offset, 0);
   expect_update_object_map(mock_local_image_ctx, mock_object_map,
@@ -546,7 +549,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, Remove) {
 
   InSequence seq;
   expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0);
-  expect_read(mock_remote_io_ctx, 0, one.range_end(), 0);
+  expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0);
   expect_write(mock_local_io_ctx, 0, one.range_end(), 0);
   expect_remove(mock_local_io_ctx, 0);
   expect_update_object_map(mock_local_image_ctx, mock_object_map,

diff --git a/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc b/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc
index 9975b5bf8157672f45941d475c93f4ba3396106f..0d1c7040497628b31974f79e83f35f2963385cb2 100644 (file)
--- a/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc
+++ b/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc
@@ -110,11 +110,10 @@ void ObjectCopyRequest<I>::send_read_object() {
         dout(20) << ": remote_snap_seq=" << remote_snap_seq << dendl;
         read_required = true;
       }
-
       dout(20) << ": read op: " << std::get<1>(sync_op) << "~"
                << std::get<2>(sync_op) << dendl;
-      op.read(std::get<1>(sync_op), std::get<2>(sync_op),
-              &std::get<3>(sync_op), nullptr);
+      op.sparse_read(std::get<1>(sync_op), std::get<2>(sync_op), &std::get<4>(sync_op),
+                     &std::get<3>(sync_op), nullptr);
       break;
     default:
       break;
@@ -173,14 +172,19 @@ void ObjectCopyRequest<I>::send_write_object() {
 
   auto &sync_ops = m_snap_sync_ops.begin()->second;
   assert(!sync_ops.empty());
-
+  uint64_t buffer_offset;
   librados::ObjectWriteOperation op;
   for (auto &sync_op : sync_ops) {
     switch (std::get<0>(sync_op)) {
     case SYNC_OP_TYPE_WRITE:
-      dout(20) << ": write op: " << std::get<1>(sync_op) << "~"
-               << std::get<3>(sync_op).length() << dendl;
-      op.write(std::get<1>(sync_op), std::get<3>(sync_op));
+      buffer_offset = 0;
+      for(auto it : std::get<4>(sync_op)){
+         bufferlist tmpbl;
+         tmpbl.substr_of(std::get<3>(sync_op), buffer_offset, it.second);
+         op.write(it.first, tmpbl);
+         buffer_offset += it.second;
+        dout(20) << ": write op: " << it.first<< "~" << it.second << dendl;
+      }
       break;
     case SYNC_OP_TYPE_TRUNC:
       dout(20) << ": trunc op: " << std::get<1>(sync_op) << dendl;
@@ -228,6 +232,7 @@ void ObjectCopyRequest<I>::handle_write_object(int r) {
 
 template <typename I>
 void ObjectCopyRequest<I>::send_update_object_map() {
+
   m_local_image_ctx->snap_lock.get_read();
   if (!m_local_image_ctx->test_features(RBD_FEATURE_OBJECT_MAP,
                                         m_local_image_ctx->snap_lock) ||
@@ -258,6 +263,7 @@ void ObjectCopyRequest<I>::send_update_object_map() {
   Context *ctx = create_context_callback<
     ObjectCopyRequest<I>, &ObjectCopyRequest<I>::handle_update_object_map>(
       this);
+
   m_local_image_ctx->object_map->aio_update(snap_object_state.first,
                                             m_object_number,
                                             m_object_number + 1,
@@ -333,20 +339,23 @@ void ObjectCopyRequest<I>::compute_diffs() {
         m_snap_sync_ops[end_remote_snap_id].emplace_back(SYNC_OP_TYPE_WRITE,
                                                          it.get_start(),
                                                          it.get_len(),
-                                                         bufferlist());
+                                                         bufferlist(),
+                                                         std::map<uint64_t, uint64_t>());
       }
       if (end_size < prev_end_size) {
         dout(20) << ": trunc op: " << end_size << dendl;
         m_snap_sync_ops[end_remote_snap_id].emplace_back(SYNC_OP_TYPE_TRUNC,
                                                          end_size, 0U,
-                                                         bufferlist());
+                                                         bufferlist(),
+                                                         std::map<uint64_t, uint64_t>());
       }
     } else {
       if (prev_exists) {
         // object remove
         dout(20) << ": remove op" << dendl;
         m_snap_sync_ops[end_remote_snap_id].emplace_back(SYNC_OP_TYPE_REMOVE,
-                                                         0U, 0U, bufferlist());
+                                                         0U, 0U, bufferlist(),
+                                                         std::map<uint64_t, uint64_t>());
       }
     }
 

diff --git a/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h b/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h
index 83fe16c52c92ccbc56766ed0314e294a9aa5b8ec..2e353383afad73a2dced7d856a2e963e20d5bfe8 100644 (file)
--- a/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h
+++ b/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h
@@ -81,7 +81,7 @@ private:
     SYNC_OP_TYPE_REMOVE
   };
 
-  typedef std::tuple<SyncOpType, uint64_t, uint64_t, bufferlist> SyncOp;
+  typedef std::tuple<SyncOpType, uint64_t, uint64_t, bufferlist, std::map<uint64_t, uint64_t> > SyncOp;
   typedef std::list<SyncOp> SyncOps;
   typedef std::map<librados::snap_t, SyncOps> SnapSyncOps;
   typedef std::map<librados::snap_t, uint8_t> SnapObjectStates;