rbd-mirror: prevent syncing to non-primary image after failover

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit beaef377d69f555277b706afff944a15086da28e)

diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
index 18d24f181153ac0bf30ece265dc10e215fd83787..c681cbfce0a57004f50fdf5e8abfa19a37907e0f 100644 (file)
--- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
+++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
@@ -312,6 +312,18 @@ public:
                                         }))));
   }
 
+  void expect_journaler_update_client(::journal::MockJournaler &mock_journaler,
+                                      const librbd::journal::ClientData &client_data,
+                                      int r) {
+    bufferlist bl;
+    ::encode(client_data, bl);
+
+    EXPECT_CALL(mock_journaler, update_client(ContentsEqual(bl), _))
+      .WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) {
+                                    m_threads->work_queue->queue(on_finish, r);
+                                  })));
+  }
+
   void expect_open_image(MockOpenImageRequest &mock_open_image_request,
                          librados::IoCtx &io_ctx, const std::string &image_id,
                          librbd::MockTestImageCtx &mock_image_ctx, int r) {
@@ -400,6 +412,63 @@ public:
 
 };
 
+TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) {
+  create_local_image();
+
+  InSequence seq;
+
+  // look up local image by global image id
+  librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
+  expect_mirror_image_get_image_id(m_local_io_ctx, "global image id",
+                                   mock_local_image_ctx.id, 0);
+
+  // lookup remote image tag class
+  cls::journal::Client client;
+  librbd::journal::ClientData client_data{
+    librbd::journal::ImageClientMeta{123}};
+  ::encode(client_data, client.data);
+  ::journal::MockJournaler mock_journaler;
+  expect_journaler_get_client(mock_journaler,
+                              librbd::Journal<>::IMAGE_CLIENT_ID,
+                              client, 0);
+
+  // lookup local peer in remote journal
+  librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
+    mock_local_image_ctx.id};
+  mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING;
+  client_data.client_meta = mirror_peer_client_meta;
+  client.data.clear();
+  ::encode(client_data, client.data);
+  expect_journaler_get_client(mock_journaler, "local mirror uuid",
+                              client, 0);
+
+  // open the remote image
+  librbd::MockJournal mock_journal;
+  librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx);
+  MockOpenImageRequest mock_open_image_request;
+  expect_open_image(mock_open_image_request, m_remote_io_ctx,
+                    mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
+  expect_journal_is_tag_owner(mock_journal, false, 0);
+
+  // switch the state to replaying
+  mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
+  client_data.client_meta = mirror_peer_client_meta;
+  expect_journaler_update_client(mock_journaler, client_data, 0);
+
+  MockCloseImageRequest mock_close_image_request;
+  expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0);
+
+  C_SaferCond ctx;
+  MockImageSyncThrottler mock_image_sync_throttler(
+    new ImageSyncThrottler<librbd::MockTestImageCtx>());
+  MockBootstrapRequest *request = create_request(
+    mock_image_sync_throttler, mock_journaler, mock_remote_image_ctx.id,
+    "global image id", "local mirror uuid", "remote mirror uuid",
+    &ctx);
+  request->send();
+  ASSERT_EQ(-EREMOTEIO, ctx.wait());
+}
+
 TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) {
   create_local_image();
 

diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
index 36294b39f054a462b7faba904dbcb3043ad1a973..2f791f0a765ca6b0cf0f54b9b806bd2ee0dc87a3 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
+++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
@@ -279,7 +279,7 @@ void BootstrapRequest<I>::handle_open_remote_image(int r) {
     dout(5) << ": remote image is not primary -- skipping image replay"
             << dendl;
     m_ret_val = -EREMOTEIO;
-    close_remote_image();
+    update_client_state();
     return;
   }
 
@@ -296,6 +296,42 @@ void BootstrapRequest<I>::handle_open_remote_image(int r) {
   open_local_image();
 }
 
+template <typename I>
+void BootstrapRequest<I>::update_client_state() {
+  if (m_client_meta->state == librbd::journal::MIRROR_PEER_STATE_REPLAYING) {
+    // state already set for replaying upon failover
+    close_remote_image();
+    return;
+  }
+
+  dout(20) << dendl;
+  update_progress("UPDATE_CLIENT_STATE");
+
+  librbd::journal::MirrorPeerClientMeta client_meta(*m_client_meta);
+  client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
+
+  librbd::journal::ClientData client_data(client_meta);
+  bufferlist data_bl;
+  ::encode(client_data, data_bl);
+
+  Context *ctx = create_context_callback<
+    BootstrapRequest<I>, &BootstrapRequest<I>::handle_update_client_state>(
+      this);
+  m_journaler->update_client(data_bl, ctx);
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_update_client_state(int r) {
+  dout(20) << ": r=" << r << dendl;
+  if (r < 0) {
+    derr << ": failed to update client: " << cpp_strerror(r) << dendl;
+  } else {
+    m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;;
+  }
+
+  close_remote_image();
+}
+
 template <typename I>
 void BootstrapRequest<I>::open_local_image() {
   dout(20) << dendl;
@@ -341,23 +377,7 @@ void BootstrapRequest<I>::handle_open_local_image(int r) {
     return;
   }
 
-  update_client();
-}
-
-template <typename I>
-void BootstrapRequest<I>::remove_local_image() {
-  dout(20) << dendl;
-
-  update_progress("REMOVE_LOCAL_IMAGE");
-
-  // TODO
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_remove_local_image(int r) {
-  dout(20) << ": r=" << r << dendl;
-
-  // TODO
+  update_client_image();
 }
 
 template <typename I>
@@ -392,10 +412,10 @@ void BootstrapRequest<I>::handle_create_local_image(int r) {
 }
 
 template <typename I>
-void BootstrapRequest<I>::update_client() {
+void BootstrapRequest<I>::update_client_image() {
   dout(20) << dendl;
 
-  update_progress("UPDATE_CLIENT");
+  update_progress("UPDATE_CLIENT_IMAGE");
 
   if (m_client_meta->image_id == (*m_local_image_ctx)->id) {
     // already registered local image with remote journal
@@ -414,13 +434,13 @@ void BootstrapRequest<I>::update_client() {
   ::encode(client_data, data_bl);
 
   Context *ctx = create_context_callback<
-    BootstrapRequest<I>, &BootstrapRequest<I>::handle_update_client>(
+    BootstrapRequest<I>, &BootstrapRequest<I>::handle_update_client_image>(
       this);
   m_journaler->update_client(data_bl, ctx);
 }
 
 template <typename I>
-void BootstrapRequest<I>::handle_update_client(int r) {
+void BootstrapRequest<I>::handle_update_client_image(int r) {
   dout(20) << ": r=" << r << dendl;
 
   if (r < 0) {
@@ -603,10 +623,6 @@ void BootstrapRequest<I>::handle_get_remote_tags(int r) {
 
 template <typename I>
 void BootstrapRequest<I>::image_sync() {
-  dout(20) << dendl;
-
-  update_progress("IMAGE_SYNC");
-
   if (m_client_meta->state == librbd::journal::MIRROR_PEER_STATE_REPLAYING) {
     // clean replay state -- no image sync required
     close_remote_image();
@@ -614,6 +630,7 @@ void BootstrapRequest<I>::image_sync() {
   }
 
   dout(20) << dendl;
+  update_progress("IMAGE_SYNC");
 
   Context *ctx = create_context_callback<
     BootstrapRequest<I>, &BootstrapRequest<I>::handle_image_sync>(

diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
index e7ef05065e9aad7accd86d8f58fae57d1b281e98..8926adf81677b8935417673d2a9abd71f2456032 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
+++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
@@ -86,48 +86,54 @@ private:
    * <start>
    *    |
    *    v
-   * GET_LOCAL_IMAGE_ID * * * * * * * * * * * *
-   *    |                                     *
-   *    v                                     *
-   * GET_REMOTE_TAG_CLASS * * * * * * * * * * *
-   *    |                                     *
-   *    v                                     *
-   * GET_CLIENT * * * * * * * * * * * * * * * *
-   *    |                                     *
-   *    v (skip if not needed)                * (error)
-   * REGISTER_CLIENT  * * * * * * * * * * * * *
-   *    |                                     *
-   *    v                                     *
-   * OPEN_REMOTE_IMAGE  * * * * * * * * * * * *
-   *    |                                     *
-   *    v                                     *
-   * OPEN_LOCAL_IMAGE * * * * * * * * * * * * *
-   *    |   .   ^                             *
-   *    |   .   |                             *
-   *    |   .   \-----------------------\     *
-   *    |   .                           |     *
-   *    |   . (image sync requested)    |     *
-   *    |   . . > REMOVE_LOCAL_IMAGE  * * * * *
-   *    |   .                   |       |     *
-   *    |   . (image doesn't    |       |     *
-   *    |   .  exist)           v       |     *
-   *    |   . . > CREATE_LOCAL_IMAGE  * * * * *
-   *    |             |                 |     *
-   *    |             \-----------------/     *
-   *    |                                     *
-   *    v (skip if not needed)                *
-   * UPDATE_CLIENT  * * * * * * * *           *
-   *    |                         *           *
-   *    v (skip if not needed)    *           *
-   * GET_REMOTE_TAGS  * * * * * * *           *
-   *    |                         *           *
-   *    v (skip if not needed)    v           *
-   * IMAGE_SYNC * * * > CLOSE_LOCAL_IMAGE     *
-   *    |                         |           *
-   *    |     /-------------------/           *
-   *    |     |                               *
-   *    v     v                               *
-   * CLOSE_REMOTE_IMAGE < * * * * * * * * * * *
+   * GET_LOCAL_IMAGE_ID * * * * * * * * * * * * * * * * *
+   *    |                                               *
+   *    v                                               *
+   * GET_REMOTE_TAG_CLASS * * * * * * * * * * * * * * * *
+   *    |                                               *
+   *    v                                               *
+   * GET_CLIENT * * * * * * * * * * * * * * * * * * * * *
+   *    |                                               *
+   *    v (skip if not needed)                          * (error)
+   * REGISTER_CLIENT  * * * * * * * * * * * * * * * * * *
+   *    |                                               *
+   *    v                                               *
+   * OPEN_REMOTE_IMAGE  * * * * * * * * * * * * * * * * *
+   *    |                                               *
+   *    | (remote image primary)                        *
+   *    \----> OPEN_LOCAL_IMAGE * * * * * * * * * * * * *
+   *    |         |   .   ^                             *
+   *    |         |   .   |                             *
+   *    |         |   .   \-----------------------\     *
+   *    |         |   .                           |     *
+   *    |         |   . (image sync requested)    |     *
+   *    |         |   . . > REMOVE_LOCAL_IMAGE  * * * * *
+   *    |         |   .                   |       |     *
+   *    |         |   . (image doesn't    |       |     *
+   *    |         |   .  exist)           v       |     *
+   *    |         |   . . > CREATE_LOCAL_IMAGE  * * * * *
+   *    |         |             |                 |     *
+   *    |         |             \-----------------/     *
+   *    |         |                                     *
+   *    |         v (skip if not needed)                *
+   *    |      UPDATE_CLIENT_IMAGE  * * * * *           *
+   *    |         |                         *           *
+   *    |         v (skip if not needed)    *           *
+   *    |      GET_REMOTE_TAGS  * * * * * * *           *
+   *    |         |                         *           *
+   *    |         v (skip if not needed)    v           *
+   *    |      IMAGE_SYNC * * * > CLOSE_LOCAL_IMAGE     *
+   *    |         |                         |           *
+   *    |         \-----------------\ /-----/           *
+   *    |                            |                  *
+   *    |                            |                  *
+   *    | (skip if not needed)       |                  *
+   *    \----> UPDATE_CLIENT_STATE  *|* * * * * * * * * *
+   *                |                |                  *
+   *    /-----------/----------------/                  *
+   *    |                                               *
+   *    v                                               *
+   * CLOSE_REMOTE_IMAGE < * * * * * * * * * * * * * * * *
    *    |
    *    v
    * <finish>
@@ -179,6 +185,9 @@ private:
   void open_remote_image();
   void handle_open_remote_image(int r);
 
+  void update_client_state();
+  void handle_update_client_state(int r);
+
   void open_local_image();
   void handle_open_local_image(int r);
 
@@ -188,8 +197,8 @@ private:
   void create_local_image();
   void handle_create_local_image(int r);
 
-  void update_client();
-  void handle_update_client(int r);
+  void update_client_image();
+  void handle_update_client_image(int r);
 
   void get_remote_tags();
   void handle_get_remote_tags(int r);