From: Jason Dillaman Date: Thu, 28 Sep 2017 18:00:29 +0000 (-0400) Subject: rbd-mirror: sync image metadata when transfering remote image X-Git-Tag: v13.0.1~721^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=9aa5dfb8ec46e8babd0f167dcc0e234a7c60a50c;p=ceph-ci.git rbd-mirror: sync image metadata when transfering remote image Fixes: http://tracker.ceph.com/issues/21535 Signed-off-by: Jason Dillaman --- diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh index 5195e6cf3e9..618ae10e6a2 100755 --- a/qa/workunits/rbd/rbd_mirror.sh +++ b/qa/workunits/rbd/rbd_mirror.sh @@ -13,6 +13,8 @@ testlog "TEST: add image and test replay" start_mirror ${CLUSTER1} image=test create_image ${CLUSTER2} ${POOL} ${image} +set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1" +set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2" wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} write_image ${CLUSTER2} ${POOL} ${image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} @@ -21,6 +23,8 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown' fi compare_images ${POOL} ${image} +compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1" +compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2" testlog "TEST: stop mirror, add image, start mirror and test replay" stop_mirror ${CLUSTER1} diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh index 325353b91bc..39db271e03f 100755 --- a/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -593,6 +593,17 @@ set_image_meta() rbd --cluster ${cluster} -p ${pool} image-meta set ${image} $key $val } +compare_image_meta() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local key=$4 + local value=$5 + + test `rbd --cluster ${cluster} -p ${pool} image-meta get ${image} ${key}` = "${value}" +} + rename_image() { local cluster=$1 diff --git a/src/test/rbd_mirror/CMakeLists.txt b/src/test/rbd_mirror/CMakeLists.txt index 47029cd9034..3b23d600d23 100644 --- a/src/test/rbd_mirror/CMakeLists.txt +++ b/src/test/rbd_mirror/CMakeLists.txt @@ -30,6 +30,7 @@ add_executable(unittest_rbd_mirror image_replayer/test_mock_PrepareLocalImageRequest.cc image_replayer/test_mock_PrepareRemoteImageRequest.cc image_sync/test_mock_ImageCopyRequest.cc + image_sync/test_mock_MetadataCopyRequest.cc image_sync/test_mock_ObjectCopyRequest.cc image_sync/test_mock_SnapshotCopyRequest.cc image_sync/test_mock_SnapshotCreateRequest.cc diff --git a/src/test/rbd_mirror/image_sync/test_mock_MetadataCopyRequest.cc b/src/test/rbd_mirror/image_sync/test_mock_MetadataCopyRequest.cc new file mode 100644 index 00000000000..35e663fac71 --- /dev/null +++ b/src/test/rbd_mirror/image_sync/test_mock_MetadataCopyRequest.cc @@ -0,0 +1,176 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "include/rbd/librbd.hpp" +#include "include/stringify.h" +#include "librbd/ImageCtx.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "tools/rbd_mirror/image_sync/MetadataCopyRequest.h" +#include + +namespace librbd { +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/image_sync/MetadataCopyRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_sync { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockImageSyncMetadataCopyRequest : public TestMockFixture { +public: + typedef MetadataCopyRequest MockMetadataCopyRequest; + typedef std::map Metadata; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_metadata_list(librbd::MockTestImageCtx &mock_image_ctx, + const Metadata& metadata, int r) { + bufferlist out_bl; + ::encode(metadata, out_bl); + + EXPECT_CALL(get_mock_io_ctx(mock_image_ctx.md_ctx), + exec(mock_image_ctx.header_oid, _, StrEq("rbd"), + StrEq("metadata_list"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(CopyInBufferlist(out_bl)), + Return(r))); + } + + void expect_metadata_set(librbd::MockTestImageCtx &mock_image_ctx, + const Metadata& metadata, int r) { + bufferlist in_bl; + ::encode(metadata, in_bl); + + EXPECT_CALL(get_mock_io_ctx(mock_image_ctx.md_ctx), + exec(mock_image_ctx.header_oid, _, StrEq("rbd"), + StrEq("metadata_set"), ContentsEqual(in_bl), _, _)) + .WillOnce(Return(r)); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::ImageCtx *m_local_image_ctx; +}; + +TEST_F(TestMockImageSyncMetadataCopyRequest, Success) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + size_t idx = 1; + Metadata key_values_1; + for (; idx <= 128; ++idx) { + bufferlist bl; + bl.append("value" + stringify(idx)); + key_values_1.emplace("key" + stringify(idx), bl); + } + + Metadata key_values_2; + for (; idx <= 255; ++idx) { + bufferlist bl; + bl.append("value" + stringify(idx)); + key_values_2.emplace("key" + stringify(idx), bl); + } + + InSequence seq; + expect_metadata_list(mock_remote_image_ctx, key_values_1, 0); + expect_metadata_set(mock_local_image_ctx, key_values_1, 0); + expect_metadata_list(mock_remote_image_ctx, key_values_2, 0); + expect_metadata_set(mock_local_image_ctx, key_values_2, 0); + + C_SaferCond ctx; + auto request = MockMetadataCopyRequest::create(&mock_local_image_ctx, + &mock_remote_image_ctx, + &ctx); + request->send(); + + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageSyncMetadataCopyRequest, Empty) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + Metadata key_values; + + InSequence seq; + expect_metadata_list(mock_remote_image_ctx, key_values, 0); + + C_SaferCond ctx; + auto request = MockMetadataCopyRequest::create(&mock_local_image_ctx, + &mock_remote_image_ctx, + &ctx); + request->send(); + + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageSyncMetadataCopyRequest, MetadataListError) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + Metadata key_values; + + InSequence seq; + expect_metadata_list(mock_remote_image_ctx, key_values, -EINVAL); + + C_SaferCond ctx; + auto request = MockMetadataCopyRequest::create(&mock_local_image_ctx, + &mock_remote_image_ctx, + &ctx); + request->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageSyncMetadataCopyRequest, MetadataSetError) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + Metadata key_values; + bufferlist bl; + bl.append("value"); + key_values.emplace("key", bl); + + InSequence seq; + expect_metadata_list(mock_remote_image_ctx, key_values, 0); + expect_metadata_set(mock_local_image_ctx, key_values, -EINVAL); + + C_SaferCond ctx; + auto request = MockMetadataCopyRequest::create(&mock_local_image_ctx, + &mock_remote_image_ctx, + &ctx); + request->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace image_sync +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_ImageSync.cc b/src/test/rbd_mirror/test_mock_ImageSync.cc index f76e02750d0..25b2cadc79f 100644 --- a/src/test/rbd_mirror/test_mock_ImageSync.cc +++ b/src/test/rbd_mirror/test_mock_ImageSync.cc @@ -12,6 +12,7 @@ #include "tools/rbd_mirror/ImageSync.h" #include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_sync/ImageCopyRequest.h" +#include "tools/rbd_mirror/image_sync/MetadataCopyRequest.h" #include "tools/rbd_mirror/image_sync/SnapshotCopyRequest.h" #include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.h" #include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.h" @@ -87,6 +88,27 @@ public: MOCK_METHOD0(send, void()); }; +template <> +class MetadataCopyRequest { +public: + static MetadataCopyRequest* s_instance; + Context *on_finish; + + static MetadataCopyRequest* create(librbd::MockTestImageCtx *local_image_ctx, + librbd::MockTestImageCtx *remote_image_ctx, + Context *on_finish) { + assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + return s_instance; + } + + MetadataCopyRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + template <> class SnapshotCopyRequest { public: @@ -166,6 +188,7 @@ public: }; ImageCopyRequest* ImageCopyRequest::s_instance = nullptr; +MetadataCopyRequest* MetadataCopyRequest::s_instance = nullptr; SnapshotCopyRequest* SnapshotCopyRequest::s_instance = nullptr; SyncPointCreateRequest* SyncPointCreateRequest::s_instance = nullptr; SyncPointPruneRequest* SyncPointPruneRequest::s_instance = nullptr; @@ -185,6 +208,7 @@ public: typedef ImageSync MockImageSync; typedef InstanceWatcher MockInstanceWatcher; typedef image_sync::ImageCopyRequest MockImageCopyRequest; + typedef image_sync::MetadataCopyRequest MockMetadataCopyRequest; typedef image_sync::SnapshotCopyRequest MockSnapshotCopyRequest; typedef image_sync::SyncPointCreateRequest MockSyncPointCreateRequest; typedef image_sync::SyncPointPruneRequest MockSyncPointPruneRequest; @@ -254,6 +278,14 @@ public: })); } + void expect_copy_metadata(MockMetadataCopyRequest &mock_metadata_copy_request, + int r) { + EXPECT_CALL(mock_metadata_copy_request, send()) + .WillOnce(Invoke([this, &mock_metadata_copy_request, r]() { + m_threads->work_queue->queue(mock_metadata_copy_request.on_finish, r); + })); + } + void expect_rollback_object_map(librbd::MockObjectMap &mock_object_map, int r) { if ((m_local_image_ctx->features & RBD_FEATURE_OBJECT_MAP) != 0) { EXPECT_CALL(mock_object_map, rollback(_, _)) @@ -321,6 +353,7 @@ TEST_F(TestMockImageSync, SimpleSync) { MockSnapshotCopyRequest mock_snapshot_copy_request; MockSyncPointCreateRequest mock_sync_point_create_request; MockSyncPointPruneRequest mock_sync_point_prune_request; + MockMetadataCopyRequest mock_metadata_copy_request; librbd::MockExclusiveLock mock_exclusive_lock; mock_local_image_ctx.exclusive_lock = &mock_exclusive_lock; @@ -339,6 +372,7 @@ TEST_F(TestMockImageSync, SimpleSync) { expect_create_object_map(mock_local_image_ctx, mock_object_map); expect_open_object_map(mock_local_image_ctx, *mock_object_map); expect_prune_sync_point(mock_sync_point_prune_request, true, 0); + expect_copy_metadata(mock_metadata_copy_request, 0); expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); C_SaferCond ctx; @@ -358,6 +392,7 @@ TEST_F(TestMockImageSync, RestartSync) { MockSnapshotCopyRequest mock_snapshot_copy_request; MockSyncPointCreateRequest mock_sync_point_create_request; MockSyncPointPruneRequest mock_sync_point_prune_request; + MockMetadataCopyRequest mock_metadata_copy_request; m_client_meta.sync_points = {{cls::rbd::UserSnapshotNamespace(), "snap1", boost::none}, {cls::rbd::UserSnapshotNamespace(), "snap2", "snap1", boost::none}}; @@ -381,6 +416,7 @@ TEST_F(TestMockImageSync, RestartSync) { expect_create_object_map(mock_local_image_ctx, mock_object_map); expect_open_object_map(mock_local_image_ctx, *mock_object_map); expect_prune_sync_point(mock_sync_point_prune_request, true, 0); + expect_copy_metadata(mock_metadata_copy_request, 0); expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); C_SaferCond ctx; diff --git a/src/test/rbd_mirror/test_mock_fixture.h b/src/test/rbd_mirror/test_mock_fixture.h index 7855f284546..0ad8c193225 100644 --- a/src/test/rbd_mirror/test_mock_fixture.h +++ b/src/test/rbd_mirror/test_mock_fixture.h @@ -21,6 +21,10 @@ namespace librbd { class MockImageCtx; } +ACTION_P(CopyInBufferlist, str) { + arg0->append(str); +} + ACTION_P(CompleteContext, r) { arg0->complete(r); } diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index 8fd1d9716cf..78a6e330d94 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -33,6 +33,7 @@ set(rbd_mirror_internal image_replayer/PrepareRemoteImageRequest.cc image_replayer/ReplayStatusFormatter.cc image_sync/ImageCopyRequest.cc + image_sync/MetadataCopyRequest.cc image_sync/ObjectCopyRequest.cc image_sync/SnapshotCopyRequest.cc image_sync/SnapshotCreateRequest.cc diff --git a/src/tools/rbd_mirror/ImageSync.cc b/src/tools/rbd_mirror/ImageSync.cc index 644ee90fad9..94df5a8aac6 100644 --- a/src/tools/rbd_mirror/ImageSync.cc +++ b/src/tools/rbd_mirror/ImageSync.cc @@ -12,6 +12,7 @@ #include "librbd/Utils.h" #include "librbd/journal/Types.h" #include "tools/rbd_mirror/image_sync/ImageCopyRequest.h" +#include "tools/rbd_mirror/image_sync/MetadataCopyRequest.h" #include "tools/rbd_mirror/image_sync/SnapshotCopyRequest.h" #include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.h" #include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.h" @@ -384,6 +385,30 @@ void ImageSync::handle_prune_sync_points(int r) { return; } + send_copy_metadata(); +} + +template +void ImageSync::send_copy_metadata() { + dout(20) << dendl; + update_progress("COPY_METADATA"); + + Context *ctx = create_context_callback< + ImageSync, &ImageSync::handle_copy_metadata>(this); + auto request = MetadataCopyRequest::create( + m_local_image_ctx, m_remote_image_ctx, ctx); + request->send(); +} + +template +void ImageSync::handle_copy_metadata(int r) { + dout(20) << ": r=" << r << dendl; + if (r < 0) { + derr << ": failed to copy metadata: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + finish(0); } diff --git a/src/tools/rbd_mirror/ImageSync.h b/src/tools/rbd_mirror/ImageSync.h index ebb156ba32f..7e836b716f7 100644 --- a/src/tools/rbd_mirror/ImageSync.h +++ b/src/tools/rbd_mirror/ImageSync.h @@ -90,10 +90,13 @@ private: * v . * REFRESH_OBJECT_MAP (skip if object . * | map disabled) . - * v + * v . * PRUNE_SYNC_POINTS . (image sync canceled) * | . * v . + * COPY_METADATA . + * | . + * v . * < . . . . . . . . . . . . . . * * @endverbatim @@ -146,6 +149,9 @@ private: void send_prune_sync_points(); void handle_prune_sync_points(int r); + void send_copy_metadata(); + void handle_copy_metadata(int r); + void update_progress(const std::string &description); }; diff --git a/src/tools/rbd_mirror/image_sync/MetadataCopyRequest.cc b/src/tools/rbd_mirror/image_sync/MetadataCopyRequest.cc new file mode 100644 index 00000000000..b9311dc64a6 --- /dev/null +++ b/src/tools/rbd_mirror/image_sync/MetadataCopyRequest.cc @@ -0,0 +1,119 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/image_sync/MetadataCopyRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/Utils.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_sync::MetadataCopyRequest: " \ + << this << " " << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_sync { + +namespace { + +const uint64_t MAX_METADATA_ITEMS = 128; + +} // anonymous namespace + +using librbd::util::create_rados_callback; + +template +void MetadataCopyRequest::send() { + list_remote_metadata(); +} + +template +void MetadataCopyRequest::list_remote_metadata() { + dout(20) << "start_key=" << m_last_metadata_key << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::metadata_list_start(&op, m_last_metadata_key, MAX_METADATA_ITEMS); + + librados::AioCompletion *aio_comp = create_rados_callback< + MetadataCopyRequest, + &MetadataCopyRequest::handle_list_remote_data>(this); + m_out_bl.clear(); + m_remote_image_ctx->md_ctx.aio_operate(m_remote_image_ctx->header_oid, + aio_comp, &op, &m_out_bl); + aio_comp->release(); +} + +template +void MetadataCopyRequest::handle_list_remote_data(int r) { + dout(20) << "r=" << r << dendl; + + Metadata metadata; + if (r == 0) { + bufferlist::iterator it = m_out_bl.begin(); + r = librbd::cls_client::metadata_list_finish(&it, &metadata); + } + + if (r < 0) { + derr << "failed to retrieve metadata: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + if (metadata.empty()) { + finish(0); + return; + } + + m_last_metadata_key = metadata.rbegin()->first; + m_more_metadata = (metadata.size() >= MAX_METADATA_ITEMS); + set_local_metadata(metadata); +} + +template +void MetadataCopyRequest::set_local_metadata(const Metadata& metadata) { + dout(20) << "count=" << metadata.size() << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::metadata_set(&op, metadata); + + librados::AioCompletion *aio_comp = create_rados_callback< + MetadataCopyRequest, + &MetadataCopyRequest::handle_set_local_metadata>(this); + m_local_image_ctx->md_ctx.aio_operate(m_local_image_ctx->header_oid, aio_comp, + &op); + aio_comp->release(); +} + +template +void MetadataCopyRequest::handle_set_local_metadata(int r) { + dout(20) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to set metadata: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + if (m_more_metadata) { + list_remote_metadata(); + return; + } + + finish(0); +} + +template +void MetadataCopyRequest::finish(int r) { + dout(20) << "r=" << r << dendl; + m_on_finish->complete(r); + delete this; +} + +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_sync::MetadataCopyRequest; diff --git a/src/tools/rbd_mirror/image_sync/MetadataCopyRequest.h b/src/tools/rbd_mirror/image_sync/MetadataCopyRequest.h new file mode 100644 index 00000000000..967b64020ae --- /dev/null +++ b/src/tools/rbd_mirror/image_sync/MetadataCopyRequest.h @@ -0,0 +1,82 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_SYNC_METADATA_COPY_REQUEST_H +#define RBD_MIRROR_IMAGE_SYNC_METADATA_COPY_REQUEST_H + +#include "include/int_types.h" +#include "include/buffer.h" +#include "include/rados/librados.hpp" +#include "librbd/ImageCtx.h" +#include +#include + +class Context; + +namespace rbd { +namespace mirror { +namespace image_sync { + +template +class MetadataCopyRequest { +public: + static MetadataCopyRequest* create(ImageCtxT *local_image_ctx, + ImageCtxT *remote_image_ctx, + Context *on_finish) { + return new MetadataCopyRequest(local_image_ctx, remote_image_ctx, + on_finish); + } + + MetadataCopyRequest(ImageCtxT *local_image_ctx, ImageCtxT *remote_image_ctx, + Context *on_finish) + : m_local_image_ctx(local_image_ctx), m_remote_image_ctx(remote_image_ctx), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * LIST_REMOTE_METADATA <-----\ + * | | (repeat if additional + * v | metadata) + * SET_LOCAL_METADATA --------/ + * | + * v + * + * + * @endverbatim + */ + typedef std::map Metadata; + + ImageCtxT *m_local_image_ctx; + ImageCtxT *m_remote_image_ctx; + Context *m_on_finish; + + bufferlist m_out_bl; + + std::string m_last_metadata_key; + bool m_more_metadata = false; + + void list_remote_metadata(); + void handle_list_remote_data(int r); + + void set_local_metadata(const Metadata& metadata); + void handle_set_local_metadata(int r); + + void finish(int r); + +}; + +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_sync::MetadataCopyRequest; + +#endif // RBD_MIRROR_IMAGE_SYNC_METADATA_COPY_REQUEST_H