From f7d43cb0dc690dd0210090276514eaf257aa22d4 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Thu, 5 Oct 2017 15:56:19 -0400 Subject: [PATCH] rbd-mirror: local primary images should still register to remote images Fixes: http://tracker.ceph.com/issues/21561 Signed-off-by: Jason Dillaman (cherry picked from commit d74b95dc10a5073ee7980583d94628ded7850ebb) --- src/test/journal/mock/MockJournaler.h | 4 + .../test_mock_PrepareRemoteImageRequest.cc | 237 +++++++++++++++++- .../rbd_mirror/test_mock_ImageReplayer.cc | 25 +- src/tools/rbd_mirror/ImageReplayer.cc | 43 ++-- src/tools/rbd_mirror/ImageReplayer.h | 2 + .../PrepareRemoteImageRequest.cc | 123 ++++++++- .../PrepareRemoteImageRequest.h | 64 ++++- 7 files changed, 464 insertions(+), 34 deletions(-) diff --git a/src/test/journal/mock/MockJournaler.h b/src/test/journal/mock/MockJournaler.h index 2182c8db4fc1a..6af3087561fba 100644 --- a/src/test/journal/mock/MockJournaler.h +++ b/src/test/journal/mock/MockJournaler.h @@ -138,6 +138,10 @@ struct MockJournaler { }; struct MockJournalerProxy { + MockJournalerProxy() { + MockJournaler::get_instance().construct(); + } + template MockJournalerProxy(IoCtxT &header_ioctx, const std::string &, const std::string &, const Settings&) { diff --git a/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc index 793811816e62d..73658b3fb8b31 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc @@ -4,8 +4,10 @@ #include "test/rbd_mirror/test_mock_fixture.h" #include "cls/rbd/cls_rbd_types.h" #include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" #include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h" +#include "test/journal/mock/MockJournaler.h" #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" #include "test/librbd/mock/MockImageCtx.h" @@ -20,10 +22,32 @@ struct MockTestImageCtx : public librbd::MockImageCtx { }; } // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits { + typedef ::journal::MockJournalerProxy Journaler; +}; + +} // namespace journal } // namespace librbd namespace rbd { namespace mirror { + +template <> +struct Threads { + Mutex &timer_lock; + SafeTimer *timer; + ContextWQ *work_queue; + + Threads(Threads *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + namespace image_replayer { template <> @@ -72,6 +96,7 @@ using ::testing::WithArg; class TestMockImageReplayerPrepareRemoteImageRequest : public TestMockFixture { public: + typedef Threads MockThreads; typedef PrepareRemoteImageRequest MockPrepareRemoteImageRequest; typedef GetMirrorImageIdRequest MockGetMirrorImageIdRequest; @@ -96,49 +121,160 @@ public: })), Return(r))); } + + void expect_journaler_get_client(::journal::MockJournaler &mock_journaler, + const std::string &client_id, + cls::journal::Client &client, int r) { + EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) { + *out_client = client; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + void expect_journaler_register_client(::journal::MockJournaler &mock_journaler, + const librbd::journal::ClientData &client_data, + int r) { + bufferlist bl; + ::encode(client_data, bl); + + EXPECT_CALL(mock_journaler, register_client(ContentsEqual(bl), _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + }))); + } }; TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, Success) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + InSequence seq; expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; expect_get_mirror_image_id(mock_get_mirror_image_id_request, "remote image id", 0); + EXPECT_CALL(mock_remote_journaler, construct()); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + cls::journal::Client client; + client.state = cls::journal::CLIENT_STATE_DISCONNECTED; + ::encode(client_data, client.data); + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, 0); + std::string remote_mirror_uuid; std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; C_SaferCond ctx; - auto req = MockPrepareRemoteImageRequest::create(m_remote_io_ctx, + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, "global image id", + "local mirror uuid", + "local image id", &remote_mirror_uuid, &remote_image_id, + &remote_journaler, + &client_state, &client_meta, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); ASSERT_EQ(std::string("remote image id"), remote_image_id); + ASSERT_TRUE(remote_journaler != nullptr); + ASSERT_EQ(cls::journal::CLIENT_STATE_DISCONNECTED, client_state); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, SuccessNotRegistered) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + EXPECT_CALL(mock_remote_journaler, construct()); + + cls::journal::Client client; + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, -ENOENT); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + expect_journaler_register_client(mock_remote_journaler, client_data, 0); + + std::string remote_mirror_uuid; + std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + "local image id", + &remote_mirror_uuid, + &remote_image_id, + &remote_journaler, + &client_state, &client_meta, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_EQ(std::string("remote image id"), remote_image_id); + ASSERT_TRUE(remote_journaler != nullptr); + ASSERT_EQ(cls::journal::CLIENT_STATE_CONNECTED, client_state); } TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorUuidError) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + InSequence seq; expect_mirror_uuid_get(m_remote_io_ctx, "", -EINVAL); std::string remote_mirror_uuid; std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; C_SaferCond ctx; - auto req = MockPrepareRemoteImageRequest::create(m_remote_io_ctx, + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, "global image id", + "local mirror uuid", + "", &remote_mirror_uuid, &remote_image_id, + &remote_journaler, + &client_state, &client_meta, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); ASSERT_EQ(std::string(""), remote_mirror_uuid); + ASSERT_TRUE(remote_journaler == nullptr); } TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdError) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + InSequence seq; expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; @@ -146,16 +282,111 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdError) { std::string remote_mirror_uuid; std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; C_SaferCond ctx; - auto req = MockPrepareRemoteImageRequest::create(m_remote_io_ctx, + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, "global image id", + "local mirror uuid", + "", &remote_mirror_uuid, &remote_image_id, + &remote_journaler, + &client_state, &client_meta, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_TRUE(remote_journaler == nullptr); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, GetClientError) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + EXPECT_CALL(mock_remote_journaler, construct()); + + cls::journal::Client client; + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, -EINVAL); + + std::string remote_mirror_uuid; + std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + "local image id", + &remote_mirror_uuid, + &remote_image_id, + &remote_journaler, + &client_state, &client_meta, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_EQ(std::string("remote image id"), remote_image_id); + ASSERT_TRUE(remote_journaler == nullptr); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, RegisterClientError) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + EXPECT_CALL(mock_remote_journaler, construct()); + + cls::journal::Client client; + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, -ENOENT); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + expect_journaler_register_client(mock_remote_journaler, client_data, -EINVAL); + + std::string remote_mirror_uuid; + std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + "local image id", + &remote_mirror_uuid, + &remote_image_id, + &remote_journaler, + &client_state, &client_meta, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_EQ(std::string("remote image id"), remote_image_id); + ASSERT_TRUE(remote_journaler == nullptr); } } // namespace image_replayer diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 9e2006c9d5190..f79dcef00fa43 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -141,16 +141,28 @@ struct PrepareRemoteImageRequest { static PrepareRemoteImageRequest* s_instance; std::string *remote_mirror_uuid = nullptr; std::string *remote_image_id = nullptr; + cls::journal::ClientState *client_state; + ::journal::MockJournalerProxy **remote_journaler = nullptr; + librbd::journal::MirrorPeerClientMeta *client_meta = nullptr; Context *on_finish = nullptr; - static PrepareRemoteImageRequest* create(librados::IoCtx &, + static PrepareRemoteImageRequest* create(Threads *threads, + librados::IoCtx &, const std::string &global_image_id, + const std::string &local_mirror_uuid, + const std::string &local_image_id, std::string *remote_mirror_uuid, std::string *remote_image_id, + ::journal::MockJournalerProxy **remote_journaler, + cls::journal::ClientState *client_state, + librbd::journal::MirrorPeerClientMeta *client_meta, Context *on_finish) { assert(s_instance != nullptr); s_instance->remote_mirror_uuid = remote_mirror_uuid; s_instance->remote_image_id = remote_image_id; + s_instance->remote_journaler = remote_journaler; + s_instance->client_state = client_state; + s_instance->client_meta = client_meta; s_instance->on_finish = on_finish; return s_instance; } @@ -422,6 +434,10 @@ public: int r) { EXPECT_CALL(mock_request, send()) .WillOnce(Invoke([&mock_request, image_id, mirror_uuid, r]() { + if (r >= 0) { + *mock_request.remote_journaler = new ::journal::MockJournalerProxy(); + } + *mock_request.remote_mirror_uuid = mirror_uuid; *mock_request.remote_image_id = image_id; mock_request.on_finish->complete(r); @@ -650,12 +666,14 @@ TEST_F(TestMockImageReplayer, LocalImagePrimary) { create_local_image(); librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + journal::MockJournaler mock_remote_journaler; MockThreads mock_threads(m_threads); expect_work_queue_repeatedly(mock_threads); expect_add_event_after_repeatedly(mock_threads); MockImageDeleter mock_image_deleter; MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; MockReplayStatusFormatter mock_replay_status_formatter; expect_get_or_send_update(mock_replay_status_formatter); @@ -664,6 +682,11 @@ TEST_F(TestMockImageReplayer, LocalImagePrimary) { expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0); expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, "", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + "remote image id", 0); + EXPECT_CALL(mock_remote_journaler, construct()); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); create_image_replayer(mock_threads, mock_image_deleter); diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index bf77e9db7bdb1..8b4c6c91c04b3 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -440,10 +440,6 @@ void ImageReplayer::handle_prepare_local_image(int r) { } else if (r < 0) { on_start_fail(r, "error preparing local image for replay"); return; - } else if (m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) { - dout(5) << "local image is primary" << dendl; - on_start_fail(0, "local image is primary"); - return; } // local image doesn't exist or is non-primary @@ -453,6 +449,11 @@ void ImageReplayer::handle_prepare_local_image(int r) { template void ImageReplayer::prepare_remote_image() { dout(20) << dendl; + if (m_peers.empty()) { + // technically nothing to bootstrap, but it handles the status update + bootstrap(); + return; + } // TODO need to support multiple remote images assert(!m_peers.empty()); @@ -461,8 +462,9 @@ void ImageReplayer::prepare_remote_image() { Context *ctx = create_context_callback< ImageReplayer, &ImageReplayer::handle_prepare_remote_image>(this); auto req = PrepareRemoteImageRequest::create( - m_remote_image.io_ctx, m_global_image_id, &m_remote_image.mirror_uuid, - &m_remote_image.image_id, ctx); + m_threads, m_remote_image.io_ctx, m_global_image_id, m_local_mirror_uuid, + m_local_image_id, &m_remote_image.mirror_uuid, &m_remote_image.image_id, + &m_remote_journaler, &m_client_state, &m_client_meta, ctx); req->send(); } @@ -470,7 +472,11 @@ template void ImageReplayer::handle_prepare_remote_image(int r) { dout(20) << "r=" << r << dendl; - if (r == -ENOENT) { + assert(r < 0 ? m_remote_journaler == nullptr : m_remote_journaler != nullptr); + if (r < 0 && !m_local_image_id.empty() && + m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) { + // local image is primary -- fall-through + } else if (r == -ENOENT) { dout(20) << "remote image does not exist" << dendl; // TODO need to support multiple remote images @@ -497,19 +503,16 @@ template void ImageReplayer::bootstrap() { dout(20) << dendl; - CephContext *cct = static_cast(m_local->cct()); - journal::Settings settings; - settings.commit_interval = cct->_conf->get_val( - "rbd_mirror_journal_commit_age"); - settings.max_fetch_bytes = cct->_conf->get_val( - "rbd_mirror_journal_max_fetch_bytes"); - - m_remote_journaler = new Journaler(m_threads->work_queue, - m_threads->timer, - &m_threads->timer_lock, - m_remote_image.io_ctx, - m_remote_image.image_id, - m_local_mirror_uuid, settings); + if (!m_local_image_id.empty() && + m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) { + dout(5) << "local image is primary" << dendl; + on_start_fail(0, "local image is primary"); + return; + } else if (m_peers.empty()) { + dout(5) << "no peer clusters" << dendl; + on_start_fail(-ENOENT, "no peer clusters"); + return; + } Context *ctx = create_context_callback< ImageReplayer, &ImageReplayer::handle_bootstrap>(this); diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index a66b02a24abc5..55e44a6d01bd1 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -331,6 +331,8 @@ private: bool m_update_status_requested = false; Context *m_on_update_status_finish = nullptr; + cls::journal::ClientState m_client_state = + cls::journal::CLIENT_STATE_DISCONNECTED; librbd::journal::MirrorPeerClientMeta m_client_meta; ReplayEntry m_replay_entry; diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc index 2e620938ff40d..35755a625a503 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc @@ -5,8 +5,13 @@ #include "include/rados/librados.hpp" #include "cls/rbd/cls_rbd_client.h" #include "common/errno.h" +#include "common/WorkQueue.h" +#include "journal/Journaler.h" +#include "journal/Settings.h" #include "librbd/ImageCtx.h" #include "librbd/Utils.h" +#include "librbd/journal/Types.h" +#include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" #define dout_context g_ceph_context @@ -20,6 +25,7 @@ namespace rbd { namespace mirror { namespace image_replayer { +using librbd::util::create_async_context_callback; using librbd::util::create_context_callback; using librbd::util::create_rados_callback; @@ -38,7 +44,7 @@ void PrepareRemoteImageRequest::get_remote_mirror_uuid() { librados::AioCompletion *aio_comp = create_rados_callback< PrepareRemoteImageRequest, &PrepareRemoteImageRequest::handle_get_remote_mirror_uuid>(this); - int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); assert(r == 0); aio_comp->release(); } @@ -75,7 +81,8 @@ void PrepareRemoteImageRequest::get_remote_image_id() { Context *ctx = create_context_callback< PrepareRemoteImageRequest, &PrepareRemoteImageRequest::handle_get_remote_image_id>(this); - auto req = GetMirrorImageIdRequest::create(m_io_ctx, m_global_image_id, + auto req = GetMirrorImageIdRequest::create(m_remote_io_ctx, + m_global_image_id, m_remote_image_id, ctx); req->send(); } @@ -90,13 +97,125 @@ void PrepareRemoteImageRequest::handle_get_remote_image_id(int r) { return; } + get_client(); +} + +template +void PrepareRemoteImageRequest::get_client() { + dout(20) << dendl; + + journal::Settings settings; + settings.commit_interval = g_ceph_context->_conf->get_val( + "rbd_mirror_journal_commit_age"); + settings.max_fetch_bytes = g_ceph_context->_conf->get_val( + "rbd_mirror_journal_max_fetch_bytes"); + + assert(*m_remote_journaler == nullptr); + *m_remote_journaler = new Journaler(m_threads->work_queue, m_threads->timer, + &m_threads->timer_lock, m_remote_io_ctx, + *m_remote_image_id, m_local_mirror_uuid, + settings); + + Context *ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + PrepareRemoteImageRequest, + &PrepareRemoteImageRequest::handle_get_client>(this)); + (*m_remote_journaler)->get_client(m_local_mirror_uuid, &m_client, ctx); +} + +template +void PrepareRemoteImageRequest::handle_get_client(int r) { + dout(20) << "r=" << r << dendl; + + if (r == -ENOENT) { + dout(10) << "client not registered" << dendl; + } else if (r < 0) { + derr << "failed to retrieve client: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + *m_client_state = m_client.state; + if (decode_client_meta()) { + // skip registration if it already exists + finish(0); + return; + } + + register_client(); +} + +template +void PrepareRemoteImageRequest::register_client() { + dout(20) << dendl; + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + m_local_image_id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + bufferlist client_data_bl; + ::encode(client_data, client_data_bl); + + Context *ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + PrepareRemoteImageRequest, + &PrepareRemoteImageRequest::handle_register_client>(this)); + (*m_remote_journaler)->register_client(client_data_bl, ctx); +} + +template +void PrepareRemoteImageRequest::handle_register_client(int r) { + dout(20) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to register with remote journal: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + *m_client_state = cls::journal::CLIENT_STATE_CONNECTED; + *m_client_meta = librbd::journal::MirrorPeerClientMeta(m_local_image_id); + m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + finish(0); } +template +bool PrepareRemoteImageRequest::decode_client_meta() { + dout(20) << dendl; + + librbd::journal::ClientData client_data; + bufferlist::iterator it = m_client.data.begin(); + try { + ::decode(client_data, it); + } catch (const buffer::error &err) { + derr << "failed to decode client meta data: " << err.what() << dendl; + return false; + } + + librbd::journal::MirrorPeerClientMeta *client_meta = + boost::get(&client_data.client_meta); + if (client_meta == nullptr) { + derr << "unknown peer registration" << dendl; + return false; + } + + *m_client_meta = *client_meta; + dout(20) << "client found: client_meta=" << *m_client_meta << dendl; + return true; +} + template void PrepareRemoteImageRequest::finish(int r) { dout(20) << "r=" << r << dendl; + if (r < 0) { + delete *m_remote_journaler; + *m_remote_journaler = nullptr; + } + m_on_finish->complete(r); delete this; } diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h index 9943fd742a7ed..8d8f553d5362e 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h @@ -5,40 +5,68 @@ #define RBD_MIRROR_IMAGE_REPLAYER_PREPARE_REMOTE_IMAGE_REQUEST_H #include "include/buffer.h" +#include "cls/journal/cls_journal_types.h" +#include "librbd/journal/TypeTraits.h" #include +namespace journal { class Journaler; } namespace librados { struct IoCtx; } namespace librbd { struct ImageCtx; } +namespace librbd { namespace journal { struct MirrorPeerClientMeta; } } struct Context; struct ContextWQ; namespace rbd { namespace mirror { + +template struct Threads; + namespace image_replayer { template class PrepareRemoteImageRequest { public: - static PrepareRemoteImageRequest *create(librados::IoCtx &io_ctx, + typedef librbd::journal::TypeTraits TypeTraits; + typedef typename TypeTraits::Journaler Journaler; + typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta; + + static PrepareRemoteImageRequest *create(Threads *threads, + librados::IoCtx &remote_io_ctx, const std::string &global_image_id, + const std::string &local_mirror_uuid, + const std::string &local_image_id, std::string *remote_mirror_uuid, std::string *remote_image_id, + Journaler **remote_journaler, + cls::journal::ClientState *client_state, + MirrorPeerClientMeta *client_meta, Context *on_finish) { - return new PrepareRemoteImageRequest(io_ctx, global_image_id, - remote_mirror_uuid, remote_image_id, - on_finish); + return new PrepareRemoteImageRequest(threads, remote_io_ctx, + global_image_id, local_mirror_uuid, + local_image_id, remote_mirror_uuid, + remote_image_id, remote_journaler, + client_state, client_meta, on_finish); } - PrepareRemoteImageRequest(librados::IoCtx &io_ctx, + PrepareRemoteImageRequest(Threads *threads, + librados::IoCtx &remote_io_ctx, const std::string &global_image_id, + const std::string &local_mirror_uuid, + const std::string &local_image_id, std::string *remote_mirror_uuid, std::string *remote_image_id, + Journaler **remote_journaler, + cls::journal::ClientState *client_state, + MirrorPeerClientMeta *client_meta, Context *on_finish) - : m_io_ctx(io_ctx), m_global_image_id(global_image_id), + : m_threads(threads), m_remote_io_ctx(remote_io_ctx), + m_global_image_id(global_image_id), + m_local_mirror_uuid(local_mirror_uuid), m_local_image_id(local_image_id), m_remote_mirror_uuid(remote_mirror_uuid), m_remote_image_id(remote_image_id), - m_on_finish(on_finish) { + m_remote_journaler(remote_journaler), m_client_state(client_state), + m_client_meta(client_meta), m_on_finish(on_finish) { } void send(); @@ -56,18 +84,31 @@ private: * GET_REMOTE_IMAGE_ID * | * v + * GET_CLIENT + * | + * v (skip if not needed) + * REGISTER_CLIENT + * | + * v * * @endverbatim */ - librados::IoCtx &m_io_ctx; + Threads *m_threads; + librados::IoCtx &m_remote_io_ctx; std::string m_global_image_id; + std::string m_local_mirror_uuid; + std::string m_local_image_id; std::string *m_remote_mirror_uuid; std::string *m_remote_image_id; + Journaler **m_remote_journaler; + cls::journal::ClientState *m_client_state; + MirrorPeerClientMeta *m_client_meta; Context *m_on_finish; bufferlist m_out_bl; + cls::journal::Client m_client; void get_remote_mirror_uuid(); void handle_get_remote_mirror_uuid(int r); @@ -75,8 +116,15 @@ private: void get_remote_image_id(); void handle_get_remote_image_id(int r); + void get_client(); + void handle_get_client(int r); + + void register_client(); + void handle_register_client(int r); + void finish(int r); + bool decode_client_meta(); }; } // namespace image_replayer -- 2.39.5