]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
rbd-mirror: local primary images should still register to remote images
authorJason Dillaman <dillaman@redhat.com>
Thu, 5 Oct 2017 19:56:19 +0000 (15:56 -0400)
committerJason Dillaman <dillaman@redhat.com>
Wed, 31 Jan 2018 16:26:47 +0000 (11:26 -0500)
Fixes: http://tracker.ceph.com/issues/21561
Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit d74b95dc10a5073ee7980583d94628ded7850ebb)

src/test/journal/mock/MockJournaler.h
src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc
src/test/rbd_mirror/test_mock_ImageReplayer.cc
src/tools/rbd_mirror/ImageReplayer.cc
src/tools/rbd_mirror/ImageReplayer.h
src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc
src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h

index 2182c8db4fc1a3471d3404b5b7fd14e18f82ee27..6af3087561fbaf94d549a69679afd0cf53e80163 100644 (file)
@@ -138,6 +138,10 @@ struct MockJournaler {
 };
 
 struct MockJournalerProxy {
+  MockJournalerProxy() {
+    MockJournaler::get_instance().construct();
+  }
+
   template <typename IoCtxT>
   MockJournalerProxy(IoCtxT &header_ioctx, const std::string &,
                      const std::string &, const Settings&) {
index 793811816e62d20421814640ccd7e6421cc624d6..73658b3fb8b31f14ab183b36c3bdbad18f784bfd 100644 (file)
@@ -4,8 +4,10 @@
 #include "test/rbd_mirror/test_mock_fixture.h"
 #include "cls/rbd/cls_rbd_types.h"
 #include "librbd/journal/TypeTraits.h"
+#include "tools/rbd_mirror/Threads.h"
 #include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h"
 #include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h"
+#include "test/journal/mock/MockJournaler.h"
 #include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
 #include "test/librbd/mock/MockImageCtx.h"
 
@@ -20,10 +22,32 @@ struct MockTestImageCtx : public librbd::MockImageCtx {
 };
 
 } // anonymous namespace
+
+namespace journal {
+
+template <>
+struct TypeTraits<MockTestImageCtx> {
+  typedef ::journal::MockJournalerProxy Journaler;
+};
+
+} // namespace journal
 } // namespace librbd
 
 namespace rbd {
 namespace mirror {
+
+template <>
+struct Threads<librbd::MockTestImageCtx> {
+  Mutex &timer_lock;
+  SafeTimer *timer;
+  ContextWQ *work_queue;
+
+  Threads(Threads<librbd::ImageCtx> *threads)
+    : timer_lock(threads->timer_lock), timer(threads->timer),
+      work_queue(threads->work_queue) {
+  }
+};
+
 namespace image_replayer {
 
 template <>
@@ -72,6 +96,7 @@ using ::testing::WithArg;
 
 class TestMockImageReplayerPrepareRemoteImageRequest : public TestMockFixture {
 public:
+  typedef Threads<librbd::MockTestImageCtx> MockThreads;
   typedef PrepareRemoteImageRequest<librbd::MockTestImageCtx> MockPrepareRemoteImageRequest;
   typedef GetMirrorImageIdRequest<librbd::MockTestImageCtx> MockGetMirrorImageIdRequest;
 
@@ -96,49 +121,160 @@ public:
                                         })),
                       Return(r)));
   }
+
+  void expect_journaler_get_client(::journal::MockJournaler &mock_journaler,
+                                   const std::string &client_id,
+                                   cls::journal::Client &client, int r) {
+    EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _))
+      .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) {
+                                          *out_client = client;
+                                        })),
+                      WithArg<2>(Invoke([this, r](Context *on_finish) {
+                                          m_threads->work_queue->queue(on_finish, r);
+                                        }))));
+  }
+
+  void expect_journaler_register_client(::journal::MockJournaler &mock_journaler,
+                                        const librbd::journal::ClientData &client_data,
+                                        int r) {
+    bufferlist bl;
+    ::encode(client_data, bl);
+
+    EXPECT_CALL(mock_journaler, register_client(ContentsEqual(bl), _))
+      .WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) {
+                                    m_threads->work_queue->queue(on_finish, r);
+                                  })));
+  }
 };
 
 TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, Success) {
+  journal::MockJournaler mock_remote_journaler;
+  MockThreads mock_threads(m_threads);
+
   InSequence seq;
   expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0);
   MockGetMirrorImageIdRequest mock_get_mirror_image_id_request;
   expect_get_mirror_image_id(mock_get_mirror_image_id_request,
                              "remote image id", 0);
 
+  EXPECT_CALL(mock_remote_journaler, construct());
+
+  librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta;
+  mirror_peer_client_meta.image_id = "local image id";
+  mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING;
+  librbd::journal::ClientData client_data{mirror_peer_client_meta};
+  cls::journal::Client client;
+  client.state = cls::journal::CLIENT_STATE_DISCONNECTED;
+  ::encode(client_data, client.data);
+  expect_journaler_get_client(mock_remote_journaler, "local mirror uuid",
+                              client, 0);
+
   std::string remote_mirror_uuid;
   std::string remote_image_id;
+  journal::MockJournalerProxy *remote_journaler = nullptr;
+  cls::journal::ClientState client_state;
+  librbd::journal::MirrorPeerClientMeta client_meta;
   C_SaferCond ctx;
-  auto req = MockPrepareRemoteImageRequest::create(m_remote_io_ctx,
+  auto req = MockPrepareRemoteImageRequest::create(&mock_threads,
+                                                   m_remote_io_ctx,
                                                    "global image id",
+                                                   "local mirror uuid",
+                                                   "local image id",
                                                    &remote_mirror_uuid,
                                                    &remote_image_id,
+                                                   &remote_journaler,
+                                                   &client_state, &client_meta,
                                                    &ctx);
   req->send();
 
   ASSERT_EQ(0, ctx.wait());
   ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid);
   ASSERT_EQ(std::string("remote image id"), remote_image_id);
+  ASSERT_TRUE(remote_journaler != nullptr);
+  ASSERT_EQ(cls::journal::CLIENT_STATE_DISCONNECTED, client_state);
+}
+
+TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, SuccessNotRegistered) {
+  journal::MockJournaler mock_remote_journaler;
+  MockThreads mock_threads(m_threads);
+
+  InSequence seq;
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0);
+  MockGetMirrorImageIdRequest mock_get_mirror_image_id_request;
+  expect_get_mirror_image_id(mock_get_mirror_image_id_request,
+                             "remote image id", 0);
+
+  EXPECT_CALL(mock_remote_journaler, construct());
+
+  cls::journal::Client client;
+  expect_journaler_get_client(mock_remote_journaler, "local mirror uuid",
+                              client, -ENOENT);
+
+  librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta;
+  mirror_peer_client_meta.image_id = "local image id";
+  mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
+  librbd::journal::ClientData client_data{mirror_peer_client_meta};
+  expect_journaler_register_client(mock_remote_journaler, client_data, 0);
+
+  std::string remote_mirror_uuid;
+  std::string remote_image_id;
+  journal::MockJournalerProxy *remote_journaler = nullptr;
+  cls::journal::ClientState client_state;
+  librbd::journal::MirrorPeerClientMeta client_meta;
+  C_SaferCond ctx;
+  auto req = MockPrepareRemoteImageRequest::create(&mock_threads,
+                                                   m_remote_io_ctx,
+                                                   "global image id",
+                                                   "local mirror uuid",
+                                                   "local image id",
+                                                   &remote_mirror_uuid,
+                                                   &remote_image_id,
+                                                   &remote_journaler,
+                                                   &client_state, &client_meta,
+                                                   &ctx);
+  req->send();
+
+  ASSERT_EQ(0, ctx.wait());
+  ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid);
+  ASSERT_EQ(std::string("remote image id"), remote_image_id);
+  ASSERT_TRUE(remote_journaler != nullptr);
+  ASSERT_EQ(cls::journal::CLIENT_STATE_CONNECTED, client_state);
 }
 
 TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorUuidError) {
+  journal::MockJournaler mock_remote_journaler;
+  MockThreads mock_threads(m_threads);
+
   InSequence seq;
   expect_mirror_uuid_get(m_remote_io_ctx, "", -EINVAL);
 
   std::string remote_mirror_uuid;
   std::string remote_image_id;
+  journal::MockJournalerProxy *remote_journaler = nullptr;
+  cls::journal::ClientState client_state;
+  librbd::journal::MirrorPeerClientMeta client_meta;
   C_SaferCond ctx;
-  auto req = MockPrepareRemoteImageRequest::create(m_remote_io_ctx,
+  auto req = MockPrepareRemoteImageRequest::create(&mock_threads,
+                                                   m_remote_io_ctx,
                                                    "global image id",
+                                                   "local mirror uuid",
+                                                   "",
                                                    &remote_mirror_uuid,
                                                    &remote_image_id,
+                                                   &remote_journaler,
+                                                   &client_state, &client_meta,
                                                    &ctx);
   req->send();
 
   ASSERT_EQ(-EINVAL, ctx.wait());
   ASSERT_EQ(std::string(""), remote_mirror_uuid);
+  ASSERT_TRUE(remote_journaler == nullptr);
 }
 
 TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdError) {
+  journal::MockJournaler mock_remote_journaler;
+  MockThreads mock_threads(m_threads);
+
   InSequence seq;
   expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0);
   MockGetMirrorImageIdRequest mock_get_mirror_image_id_request;
@@ -146,16 +282,111 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdError) {
 
   std::string remote_mirror_uuid;
   std::string remote_image_id;
+  journal::MockJournalerProxy *remote_journaler = nullptr;
+  cls::journal::ClientState client_state;
+  librbd::journal::MirrorPeerClientMeta client_meta;
   C_SaferCond ctx;
-  auto req = MockPrepareRemoteImageRequest::create(m_remote_io_ctx,
+  auto req = MockPrepareRemoteImageRequest::create(&mock_threads,
+                                                   m_remote_io_ctx,
                                                    "global image id",
+                                                   "local mirror uuid",
+                                                   "",
                                                    &remote_mirror_uuid,
                                                    &remote_image_id,
+                                                   &remote_journaler,
+                                                   &client_state, &client_meta,
                                                    &ctx);
   req->send();
 
   ASSERT_EQ(-EINVAL, ctx.wait());
   ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid);
+  ASSERT_TRUE(remote_journaler == nullptr);
+}
+
+TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, GetClientError) {
+  journal::MockJournaler mock_remote_journaler;
+  MockThreads mock_threads(m_threads);
+
+  InSequence seq;
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0);
+  MockGetMirrorImageIdRequest mock_get_mirror_image_id_request;
+  expect_get_mirror_image_id(mock_get_mirror_image_id_request,
+                             "remote image id", 0);
+
+  EXPECT_CALL(mock_remote_journaler, construct());
+
+  cls::journal::Client client;
+  expect_journaler_get_client(mock_remote_journaler, "local mirror uuid",
+                              client, -EINVAL);
+
+  std::string remote_mirror_uuid;
+  std::string remote_image_id;
+  journal::MockJournalerProxy *remote_journaler = nullptr;
+  cls::journal::ClientState client_state;
+  librbd::journal::MirrorPeerClientMeta client_meta;
+  C_SaferCond ctx;
+  auto req = MockPrepareRemoteImageRequest::create(&mock_threads,
+                                                   m_remote_io_ctx,
+                                                   "global image id",
+                                                   "local mirror uuid",
+                                                   "local image id",
+                                                   &remote_mirror_uuid,
+                                                   &remote_image_id,
+                                                   &remote_journaler,
+                                                   &client_state, &client_meta,
+                                                   &ctx);
+  req->send();
+
+  ASSERT_EQ(-EINVAL, ctx.wait());
+  ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid);
+  ASSERT_EQ(std::string("remote image id"), remote_image_id);
+  ASSERT_TRUE(remote_journaler == nullptr);
+}
+
+TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, RegisterClientError) {
+  journal::MockJournaler mock_remote_journaler;
+  MockThreads mock_threads(m_threads);
+
+  InSequence seq;
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0);
+  MockGetMirrorImageIdRequest mock_get_mirror_image_id_request;
+  expect_get_mirror_image_id(mock_get_mirror_image_id_request,
+                             "remote image id", 0);
+
+  EXPECT_CALL(mock_remote_journaler, construct());
+
+  cls::journal::Client client;
+  expect_journaler_get_client(mock_remote_journaler, "local mirror uuid",
+                              client, -ENOENT);
+
+  librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta;
+  mirror_peer_client_meta.image_id = "local image id";
+  mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
+  librbd::journal::ClientData client_data{mirror_peer_client_meta};
+  expect_journaler_register_client(mock_remote_journaler, client_data, -EINVAL);
+
+  std::string remote_mirror_uuid;
+  std::string remote_image_id;
+  journal::MockJournalerProxy *remote_journaler = nullptr;
+  cls::journal::ClientState client_state;
+  librbd::journal::MirrorPeerClientMeta client_meta;
+  C_SaferCond ctx;
+  auto req = MockPrepareRemoteImageRequest::create(&mock_threads,
+                                                   m_remote_io_ctx,
+                                                   "global image id",
+                                                   "local mirror uuid",
+                                                   "local image id",
+                                                   &remote_mirror_uuid,
+                                                   &remote_image_id,
+                                                   &remote_journaler,
+                                                   &client_state, &client_meta,
+                                                   &ctx);
+  req->send();
+
+  ASSERT_EQ(-EINVAL, ctx.wait());
+  ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid);
+  ASSERT_EQ(std::string("remote image id"), remote_image_id);
+  ASSERT_TRUE(remote_journaler == nullptr);
 }
 
 } // namespace image_replayer
index 9e2006c9d5190a250dd6243d6e0e8ab1097332f2..f79dcef00fa432d6414e9ff36f10f94ee1f626dd 100644 (file)
@@ -141,16 +141,28 @@ struct PrepareRemoteImageRequest<librbd::MockTestImageCtx> {
   static PrepareRemoteImageRequest* s_instance;
   std::string *remote_mirror_uuid = nullptr;
   std::string *remote_image_id = nullptr;
+  cls::journal::ClientState *client_state;
+  ::journal::MockJournalerProxy **remote_journaler = nullptr;
+  librbd::journal::MirrorPeerClientMeta *client_meta = nullptr;
   Context *on_finish = nullptr;
 
-  static PrepareRemoteImageRequest* create(librados::IoCtx &,
+  static PrepareRemoteImageRequest* create(Threads<librbd::MockTestImageCtx> *threads,
+                                           librados::IoCtx &,
                                            const std::string &global_image_id,
+                                           const std::string &local_mirror_uuid,
+                                           const std::string &local_image_id,
                                            std::string *remote_mirror_uuid,
                                            std::string *remote_image_id,
+                                           ::journal::MockJournalerProxy **remote_journaler,
+                                           cls::journal::ClientState *client_state,
+                                           librbd::journal::MirrorPeerClientMeta *client_meta,
                                            Context *on_finish) {
     assert(s_instance != nullptr);
     s_instance->remote_mirror_uuid = remote_mirror_uuid;
     s_instance->remote_image_id = remote_image_id;
+    s_instance->remote_journaler = remote_journaler;
+    s_instance->client_state = client_state;
+    s_instance->client_meta = client_meta;
     s_instance->on_finish = on_finish;
     return s_instance;
   }
@@ -422,6 +434,10 @@ public:
                    int r) {
     EXPECT_CALL(mock_request, send())
       .WillOnce(Invoke([&mock_request, image_id, mirror_uuid, r]() {
+                  if (r >= 0) {
+                    *mock_request.remote_journaler = new ::journal::MockJournalerProxy();
+                  }
+
                   *mock_request.remote_mirror_uuid = mirror_uuid;
                   *mock_request.remote_image_id = image_id;
                   mock_request.on_finish->complete(r);
@@ -650,12 +666,14 @@ TEST_F(TestMockImageReplayer, LocalImagePrimary) {
   create_local_image();
   librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
 
+  journal::MockJournaler mock_remote_journaler;
   MockThreads mock_threads(m_threads);
   expect_work_queue_repeatedly(mock_threads);
   expect_add_event_after_repeatedly(mock_threads);
 
   MockImageDeleter mock_image_deleter;
   MockPrepareLocalImageRequest mock_prepare_local_image_request;
+  MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
   MockReplayStatusFormatter mock_replay_status_formatter;
 
   expect_get_or_send_update(mock_replay_status_formatter);
@@ -664,6 +682,11 @@ TEST_F(TestMockImageReplayer, LocalImagePrimary) {
   expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
               "", 0);
+  expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
+              "remote image id", 0);
+  EXPECT_CALL(mock_remote_journaler, construct());
+  EXPECT_CALL(mock_remote_journaler, remove_listener(_));
+  expect_shut_down(mock_remote_journaler, 0);
 
   create_image_replayer(mock_threads, mock_image_deleter);
 
index bf77e9db7bdb1b5832626b1615c1b5a2e2a9852b..8b4c6c91c04b3f3f646eb974e6e2e0ca5819df03 100644 (file)
@@ -440,10 +440,6 @@ void ImageReplayer<I>::handle_prepare_local_image(int r) {
   } else if (r < 0) {
     on_start_fail(r, "error preparing local image for replay");
     return;
-  } else if (m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) {
-    dout(5) << "local image is primary" << dendl;
-    on_start_fail(0, "local image is primary");
-    return;
   }
 
   // local image doesn't exist or is non-primary
@@ -453,6 +449,11 @@ void ImageReplayer<I>::handle_prepare_local_image(int r) {
 template <typename I>
 void ImageReplayer<I>::prepare_remote_image() {
   dout(20) << dendl;
+  if (m_peers.empty()) {
+    // technically nothing to bootstrap, but it handles the status update
+    bootstrap();
+    return;
+  }
 
   // TODO need to support multiple remote images
   assert(!m_peers.empty());
@@ -461,8 +462,9 @@ void ImageReplayer<I>::prepare_remote_image() {
   Context *ctx = create_context_callback<
     ImageReplayer, &ImageReplayer<I>::handle_prepare_remote_image>(this);
   auto req = PrepareRemoteImageRequest<I>::create(
-    m_remote_image.io_ctx, m_global_image_id, &m_remote_image.mirror_uuid,
-    &m_remote_image.image_id, ctx);
+    m_threads, m_remote_image.io_ctx, m_global_image_id, m_local_mirror_uuid,
+    m_local_image_id, &m_remote_image.mirror_uuid, &m_remote_image.image_id,
+    &m_remote_journaler, &m_client_state, &m_client_meta, ctx);
   req->send();
 }
 
@@ -470,7 +472,11 @@ template <typename I>
 void ImageReplayer<I>::handle_prepare_remote_image(int r) {
   dout(20) << "r=" << r << dendl;
 
-  if (r == -ENOENT) {
+  assert(r < 0 ? m_remote_journaler == nullptr : m_remote_journaler != nullptr);
+  if (r < 0 && !m_local_image_id.empty() &&
+      m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) {
+    // local image is primary -- fall-through
+  } else if (r == -ENOENT) {
     dout(20) << "remote image does not exist" << dendl;
 
     // TODO need to support multiple remote images
@@ -497,19 +503,16 @@ template <typename I>
 void ImageReplayer<I>::bootstrap() {
   dout(20) << dendl;
 
-  CephContext *cct = static_cast<CephContext *>(m_local->cct());
-  journal::Settings settings;
-  settings.commit_interval = cct->_conf->get_val<double>(
-    "rbd_mirror_journal_commit_age");
-  settings.max_fetch_bytes = cct->_conf->get_val<uint64_t>(
-    "rbd_mirror_journal_max_fetch_bytes");
-
-  m_remote_journaler = new Journaler(m_threads->work_queue,
-                                     m_threads->timer,
-                                     &m_threads->timer_lock,
-                                     m_remote_image.io_ctx,
-                                     m_remote_image.image_id,
-                                     m_local_mirror_uuid, settings);
+  if (!m_local_image_id.empty() &&
+      m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) {
+    dout(5) << "local image is primary" << dendl;
+    on_start_fail(0, "local image is primary");
+    return;
+  } else if (m_peers.empty()) {
+    dout(5) << "no peer clusters" << dendl;
+    on_start_fail(-ENOENT, "no peer clusters");
+    return;
+  }
 
   Context *ctx = create_context_callback<
     ImageReplayer, &ImageReplayer<I>::handle_bootstrap>(this);
index a66b02a24abc553c82ceaf346548656983367945..55e44a6d01bd1979904d9c23f79ef5f3b92ee598 100644 (file)
@@ -331,6 +331,8 @@ private:
   bool m_update_status_requested = false;
   Context *m_on_update_status_finish = nullptr;
 
+  cls::journal::ClientState m_client_state =
+    cls::journal::CLIENT_STATE_DISCONNECTED;
   librbd::journal::MirrorPeerClientMeta m_client_meta;
 
   ReplayEntry m_replay_entry;
index 2e620938ff40d8d982c43160a69c6e8ee4b4ed34..35755a625a503bdb36c73cc8b49ff82924d3fc41 100644 (file)
@@ -5,8 +5,13 @@
 #include "include/rados/librados.hpp"
 #include "cls/rbd/cls_rbd_client.h"
 #include "common/errno.h"
+#include "common/WorkQueue.h"
+#include "journal/Journaler.h"
+#include "journal/Settings.h"
 #include "librbd/ImageCtx.h"
 #include "librbd/Utils.h"
+#include "librbd/journal/Types.h"
+#include "tools/rbd_mirror/Threads.h"
 #include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h"
 
 #define dout_context g_ceph_context
@@ -20,6 +25,7 @@ namespace rbd {
 namespace mirror {
 namespace image_replayer {
 
+using librbd::util::create_async_context_callback;
 using librbd::util::create_context_callback;
 using librbd::util::create_rados_callback;
 
@@ -38,7 +44,7 @@ void PrepareRemoteImageRequest<I>::get_remote_mirror_uuid() {
   librados::AioCompletion *aio_comp = create_rados_callback<
     PrepareRemoteImageRequest<I>,
     &PrepareRemoteImageRequest<I>::handle_get_remote_mirror_uuid>(this);
-  int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl);
+  int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl);
   assert(r == 0);
   aio_comp->release();
 }
@@ -75,7 +81,8 @@ void PrepareRemoteImageRequest<I>::get_remote_image_id() {
   Context *ctx = create_context_callback<
     PrepareRemoteImageRequest<I>,
     &PrepareRemoteImageRequest<I>::handle_get_remote_image_id>(this);
-  auto req = GetMirrorImageIdRequest<I>::create(m_io_ctx, m_global_image_id,
+  auto req = GetMirrorImageIdRequest<I>::create(m_remote_io_ctx,
+                                                m_global_image_id,
                                                 m_remote_image_id, ctx);
   req->send();
 }
@@ -90,13 +97,125 @@ void PrepareRemoteImageRequest<I>::handle_get_remote_image_id(int r) {
     return;
   }
 
+  get_client();
+}
+
+template <typename I>
+void PrepareRemoteImageRequest<I>::get_client() {
+  dout(20) << dendl;
+
+  journal::Settings settings;
+  settings.commit_interval = g_ceph_context->_conf->get_val<double>(
+    "rbd_mirror_journal_commit_age");
+  settings.max_fetch_bytes = g_ceph_context->_conf->get_val<uint64_t>(
+    "rbd_mirror_journal_max_fetch_bytes");
+
+  assert(*m_remote_journaler == nullptr);
+  *m_remote_journaler = new Journaler(m_threads->work_queue, m_threads->timer,
+                                      &m_threads->timer_lock, m_remote_io_ctx,
+                                      *m_remote_image_id, m_local_mirror_uuid,
+                                      settings);
+
+  Context *ctx = create_async_context_callback(
+    m_threads->work_queue, create_context_callback<
+      PrepareRemoteImageRequest<I>,
+      &PrepareRemoteImageRequest<I>::handle_get_client>(this));
+  (*m_remote_journaler)->get_client(m_local_mirror_uuid, &m_client, ctx);
+}
+
+template <typename I>
+void PrepareRemoteImageRequest<I>::handle_get_client(int r) {
+  dout(20) << "r=" << r << dendl;
+
+  if (r == -ENOENT) {
+    dout(10) << "client not registered" << dendl;
+  } else if (r < 0) {
+    derr << "failed to retrieve client: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  *m_client_state = m_client.state;
+  if (decode_client_meta()) {
+    // skip registration if it already exists
+    finish(0);
+    return;
+  }
+
+  register_client();
+}
+
+template <typename I>
+void PrepareRemoteImageRequest<I>::register_client() {
+  dout(20) << dendl;
+
+  librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
+    m_local_image_id};
+  mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
+
+  librbd::journal::ClientData client_data{mirror_peer_client_meta};
+  bufferlist client_data_bl;
+  ::encode(client_data, client_data_bl);
+
+  Context *ctx = create_async_context_callback(
+    m_threads->work_queue, create_context_callback<
+      PrepareRemoteImageRequest<I>,
+      &PrepareRemoteImageRequest<I>::handle_register_client>(this));
+  (*m_remote_journaler)->register_client(client_data_bl, ctx);
+}
+
+template <typename I>
+void PrepareRemoteImageRequest<I>::handle_register_client(int r) {
+  dout(20) << "r=" << r << dendl;
+
+  if (r < 0) {
+    derr << "failed to register with remote journal: " << cpp_strerror(r)
+         << dendl;
+    finish(r);
+    return;
+  }
+
+  *m_client_state = cls::journal::CLIENT_STATE_CONNECTED;
+  *m_client_meta = librbd::journal::MirrorPeerClientMeta(m_local_image_id);
+  m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
+
   finish(0);
 }
 
+template <typename I>
+bool PrepareRemoteImageRequest<I>::decode_client_meta() {
+  dout(20) << dendl;
+
+  librbd::journal::ClientData client_data;
+  bufferlist::iterator it = m_client.data.begin();
+  try {
+    ::decode(client_data, it);
+  } catch (const buffer::error &err) {
+    derr << "failed to decode client meta data: " << err.what() << dendl;
+    return false;
+  }
+
+  librbd::journal::MirrorPeerClientMeta *client_meta =
+    boost::get<librbd::journal::MirrorPeerClientMeta>(&client_data.client_meta);
+  if (client_meta == nullptr) {
+    derr << "unknown peer registration" << dendl;
+    return false;
+  }
+
+  *m_client_meta = *client_meta;
+  dout(20) << "client found: client_meta=" << *m_client_meta << dendl;
+  return true;
+}
+
 template <typename I>
 void PrepareRemoteImageRequest<I>::finish(int r) {
   dout(20) << "r=" << r << dendl;
 
+  if (r < 0) {
+    delete *m_remote_journaler;
+    *m_remote_journaler = nullptr;
+  }
+
   m_on_finish->complete(r);
   delete this;
 }
index 9943fd742a7ed4fc5b142d4c3f999a0064b1e25a..8d8f553d5362efc3cb77d779cd715ee76ad16e28 100644 (file)
@@ -5,40 +5,68 @@
 #define RBD_MIRROR_IMAGE_REPLAYER_PREPARE_REMOTE_IMAGE_REQUEST_H
 
 #include "include/buffer.h"
+#include "cls/journal/cls_journal_types.h"
+#include "librbd/journal/TypeTraits.h"
 #include <string>
 
+namespace journal { class Journaler; }
 namespace librados { struct IoCtx; }
 namespace librbd { struct ImageCtx; }
+namespace librbd { namespace journal { struct MirrorPeerClientMeta; } }
 
 struct Context;
 struct ContextWQ;
 
 namespace rbd {
 namespace mirror {
+
+template <typename> struct Threads;
+
 namespace image_replayer {
 
 template <typename ImageCtxT = librbd::ImageCtx>
 class PrepareRemoteImageRequest {
 public:
-  static PrepareRemoteImageRequest *create(librados::IoCtx &io_ctx,
+  typedef librbd::journal::TypeTraits<ImageCtxT> TypeTraits;
+  typedef typename TypeTraits::Journaler Journaler;
+  typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta;
+
+  static PrepareRemoteImageRequest *create(Threads<ImageCtxT> *threads,
+                                           librados::IoCtx &remote_io_ctx,
                                            const std::string &global_image_id,
+                                           const std::string &local_mirror_uuid,
+                                           const std::string &local_image_id,
                                            std::string *remote_mirror_uuid,
                                            std::string *remote_image_id,
+                                           Journaler **remote_journaler,
+                                           cls::journal::ClientState *client_state,
+                                           MirrorPeerClientMeta *client_meta,
                                            Context *on_finish) {
-    return new PrepareRemoteImageRequest(io_ctx, global_image_id,
-                                         remote_mirror_uuid, remote_image_id,
-                                         on_finish);
+    return new PrepareRemoteImageRequest(threads, remote_io_ctx,
+                                         global_image_id, local_mirror_uuid,
+                                         local_image_id, remote_mirror_uuid,
+                                         remote_image_id, remote_journaler,
+                                         client_state, client_meta, on_finish);
   }
 
-  PrepareRemoteImageRequest(librados::IoCtx &io_ctx,
+  PrepareRemoteImageRequest(Threads<ImageCtxT> *threads,
+                           librados::IoCtx &remote_io_ctx,
                            const std::string &global_image_id,
+                           const std::string &local_mirror_uuid,
+                           const std::string &local_image_id,
                            std::string *remote_mirror_uuid,
                            std::string *remote_image_id,
+                           Journaler **remote_journaler,
+                           cls::journal::ClientState *client_state,
+                           MirrorPeerClientMeta *client_meta,
                            Context *on_finish)
-    : m_io_ctx(io_ctx), m_global_image_id(global_image_id),
+    : m_threads(threads), m_remote_io_ctx(remote_io_ctx),
+      m_global_image_id(global_image_id),
+      m_local_mirror_uuid(local_mirror_uuid), m_local_image_id(local_image_id),
       m_remote_mirror_uuid(remote_mirror_uuid),
       m_remote_image_id(remote_image_id),
-      m_on_finish(on_finish) {
+      m_remote_journaler(remote_journaler), m_client_state(client_state),
+      m_client_meta(client_meta), m_on_finish(on_finish) {
   }
 
   void send();
@@ -56,18 +84,31 @@ private:
    * GET_REMOTE_IMAGE_ID
    *    |
    *    v
+   * GET_CLIENT
+   *    |
+   *    v (skip if not needed)
+   * REGISTER_CLIENT
+   *    |
+   *    v
    * <finish>
 
    * @endverbatim
    */
 
-  librados::IoCtx &m_io_ctx;
+  Threads<ImageCtxT> *m_threads;
+  librados::IoCtx &m_remote_io_ctx;
   std::string m_global_image_id;
+  std::string m_local_mirror_uuid;
+  std::string m_local_image_id;
   std::string *m_remote_mirror_uuid;
   std::string *m_remote_image_id;
+  Journaler **m_remote_journaler;
+  cls::journal::ClientState *m_client_state;
+  MirrorPeerClientMeta *m_client_meta;
   Context *m_on_finish;
 
   bufferlist m_out_bl;
+  cls::journal::Client m_client;
 
   void get_remote_mirror_uuid();
   void handle_get_remote_mirror_uuid(int r);
@@ -75,8 +116,15 @@ private:
   void get_remote_image_id();
   void handle_get_remote_image_id(int r);
 
+  void get_client();
+  void handle_get_client(int r);
+
+  void register_client();
+  void handle_register_client(int r);
+
   void finish(int r);
 
+  bool decode_client_meta();
 };
 
 } // namespace image_replayer