From: Mykola Golub <mgolub@mirantis.com>
Date: Mon, 9 Jan 2017 08:23:19 +0000 (+0100)
Subject: rbd-mirror: async request to test if image is primary
X-Git-Tag: v11.2.1~80^2~3
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=d5fa6a65b52c3f2cf5363ebb55af7e96d8be90ab;p=ceph.git

rbd-mirror: async request to test if image is primary

Signed-off-by: Mykola Golub <mgolub@mirantis.com>
(cherry picked from commit 0a1cb35caacdf85029f31a0364dc07a5d7462f5f)
---

diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
index 56bb21f2df279..542b78c54289c 100644
--- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
+++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
@@ -9,6 +9,7 @@
 #include "tools/rbd_mirror/image_replayer/BootstrapRequest.h"
 #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h"
 #include "tools/rbd_mirror/image_replayer/CreateImageRequest.h"
+#include "tools/rbd_mirror/image_replayer/IsPrimaryRequest.h"
 #include "tools/rbd_mirror/image_replayer/OpenImageRequest.h"
 #include "tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h"
 #include "test/journal/mock/MockJournaler.h"
@@ -149,6 +150,31 @@ struct CreateImageRequest<librbd::MockTestImageCtx> {
   MOCK_METHOD0(send, void());
 };
 
+template<>
+struct IsPrimaryRequest<librbd::MockTestImageCtx> {
+  static IsPrimaryRequest* s_instance;
+  bool *primary = nullptr;
+  Context *on_finish = nullptr;
+
+  static IsPrimaryRequest* create(librbd::MockTestImageCtx *image_ctx,
+                                  bool *primary, Context *on_finish) {
+    assert(s_instance != nullptr);
+    s_instance->primary = primary;
+    s_instance->on_finish = on_finish;
+    return s_instance;
+  }
+
+  IsPrimaryRequest() {
+    assert(s_instance == nullptr);
+    s_instance = this;
+  }
+  ~IsPrimaryRequest() {
+    s_instance = nullptr;
+  }
+
+  MOCK_METHOD0(send, void());
+};
+
 template<>
 struct OpenImageRequest<librbd::MockTestImageCtx> {
   static OpenImageRequest* s_instance;
@@ -216,6 +242,8 @@ CloseImageRequest<librbd::MockTestImageCtx>*
   CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 CreateImageRequest<librbd::MockTestImageCtx>*
   CreateImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
+IsPrimaryRequest<librbd::MockTestImageCtx>*
+  IsPrimaryRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 OpenImageRequest<librbd::MockTestImageCtx>*
   OpenImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 OpenLocalImageRequest<librbd::MockTestImageCtx>*
@@ -251,6 +279,7 @@ public:
   typedef ImageSyncThrottlerRef<librbd::MockTestImageCtx> MockImageSyncThrottler;
   typedef BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest;
   typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest;
+  typedef IsPrimaryRequest<librbd::MockTestImageCtx> MockIsPrimaryRequest;
   typedef OpenImageRequest<librbd::MockTestImageCtx> MockOpenImageRequest;
   typedef OpenLocalImageRequest<librbd::MockTestImageCtx> MockOpenLocalImageRequest;
   typedef std::list<cls::journal::Tag> Tags;
@@ -358,11 +387,13 @@ public:
         }));
   }
 
-  void expect_journal_is_tag_owner(librbd::MockJournal &mock_journal,
-                                   bool is_owner, int r) {
-    EXPECT_CALL(mock_journal, is_tag_owner(_))
-      .WillOnce(DoAll(SetArgPointee<0>(is_owner),
-                      Return(r)));
+  void expect_is_primary(MockIsPrimaryRequest &mock_is_primary_request,
+			 bool primary, int r) {
+    EXPECT_CALL(mock_is_primary_request, send())
+      .WillOnce(Invoke([this, &mock_is_primary_request, primary, r]() {
+          *mock_is_primary_request.primary = primary;
+          m_threads->work_queue->queue(mock_is_primary_request.on_finish, r);
+        }));
   }
 
   void expect_journal_get_tag_tid(librbd::MockJournal &mock_journal,
@@ -455,7 +486,8 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) {
   MockOpenImageRequest mock_open_image_request;
   expect_open_image(mock_open_image_request, m_remote_io_ctx,
                     mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
-  expect_journal_is_tag_owner(mock_journal, false, 0);
+  MockIsPrimaryRequest mock_is_primary_request;
+  expect_is_primary(mock_is_primary_request, false, 0);
 
   // switch the state to replaying
   mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
@@ -512,7 +544,8 @@ TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) {
   MockOpenImageRequest mock_open_image_request;
   expect_open_image(mock_open_image_request, m_remote_io_ctx,
                     mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
-  expect_journal_is_tag_owner(mock_journal, true, 0);
+  MockIsPrimaryRequest mock_is_primary_request;
+  expect_is_primary(mock_is_primary_request, true, 0);
 
   // open the local image
   mock_local_image_ctx.journal = &mock_journal;
@@ -590,7 +623,8 @@ TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) {
   MockOpenImageRequest mock_open_image_request;
   expect_open_image(mock_open_image_request, m_remote_io_ctx,
                     mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
-  expect_journal_is_tag_owner(mock_journal, true, 0);
+  MockIsPrimaryRequest mock_is_primary_request;
+  expect_is_primary(mock_is_primary_request, true, 0);
 
   // open the local image
   mock_local_image_ctx.journal = &mock_journal;
@@ -678,7 +712,8 @@ TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) {
   MockOpenImageRequest mock_open_image_request;
   expect_open_image(mock_open_image_request, m_remote_io_ctx,
                     mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
-  expect_journal_is_tag_owner(mock_journal, true, 0);
+  MockIsPrimaryRequest mock_is_primary_request;
+  expect_is_primary(mock_is_primary_request, true, 0);
 
   // open the local image
   mock_local_image_ctx.journal = &mock_journal;
@@ -754,7 +789,8 @@ TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) {
   MockOpenImageRequest mock_open_image_request;
   expect_open_image(mock_open_image_request, m_remote_io_ctx,
                     mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
-  expect_journal_is_tag_owner(mock_journal, true, 0);
+  MockIsPrimaryRequest mock_is_primary_request;
+  expect_is_primary(mock_is_primary_request, true, 0);
 
   // open the local image
   mock_local_image_ctx.journal = &mock_journal;
diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt
index fc7b504c51e2d..fff32f7168dd5 100644
--- a/src/tools/rbd_mirror/CMakeLists.txt
+++ b/src/tools/rbd_mirror/CMakeLists.txt
@@ -13,6 +13,7 @@ set(rbd_mirror_internal
   image_replayer/CloseImageRequest.cc
   image_replayer/CreateImageRequest.cc
   image_replayer/EventPreprocessor.cc
+  image_replayer/IsPrimaryRequest.cc
   image_replayer/OpenImageRequest.cc
   image_replayer/OpenLocalImageRequest.cc
   image_replayer/ReplayStatusFormatter.cc
diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
index 20ddf90da613b..54c3818e93aad 100644
--- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
+++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
@@ -5,6 +5,7 @@
 #include "BootstrapRequest.h"
 #include "CloseImageRequest.h"
 #include "CreateImageRequest.h"
+#include "IsPrimaryRequest.h"
 #include "OpenImageRequest.h"
 #include "OpenLocalImageRequest.h"
 #include "common/debug.h"
@@ -257,11 +258,6 @@ void BootstrapRequest<I>::open_remote_image() {
 
 template <typename I>
 void BootstrapRequest<I>::handle_open_remote_image(int r) {
-  // deduce the class type for the journal to support unit tests
-  using Journal = typename std::decay<
-    typename std::remove_pointer<decltype(std::declval<I>().journal)>
-    ::type>::type;
-
   dout(20) << ": r=" << r << dendl;
 
   if (r < 0) {
@@ -271,18 +267,36 @@ void BootstrapRequest<I>::handle_open_remote_image(int r) {
     return;
   }
 
-  // TODO: make async
-  bool tag_owner;
-  r = Journal::is_tag_owner(m_remote_image_ctx, &tag_owner);
+  is_primary();
+}
+
+template <typename I>
+void BootstrapRequest<I>::is_primary() {
+  dout(20) << dendl;
+
+  update_progress("OPEN_REMOTE_IMAGE");
+
+  Context *ctx = create_context_callback<
+    BootstrapRequest<I>, &BootstrapRequest<I>::handle_is_primary>(
+      this);
+  IsPrimaryRequest<I> *request = IsPrimaryRequest<I>::create(m_remote_image_ctx,
+                                                             &m_primary, ctx);
+  request->send();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_is_primary(int r) {
+  dout(20) << ": r=" << r << dendl;
+
   if (r < 0) {
-    derr << ": failed to query remote image primary status: " << cpp_strerror(r)
+    derr << ": error querying remote image primary status: " << cpp_strerror(r)
          << dendl;
     m_ret_val = r;
     close_remote_image();
     return;
   }
 
-  if (!tag_owner) {
+  if (!m_primary) {
     dout(5) << ": remote image is not primary -- skipping image replay"
             << dendl;
     m_ret_val = -EREMOTEIO;
diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
index 51d394e3d1582..6b4ed215a0f26 100644
--- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
+++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
@@ -101,6 +101,9 @@ private:
    *    v                                               *
    * OPEN_REMOTE_IMAGE  * * * * * * * * * * * * * * * * *
    *    |                                               *
+   *    v                                               *
+   * IS_PRIMARY * * * * * * * * * * * * * * * * * * * * *
+   *    |                                               *
    *    | (remote image primary)                        *
    *    \----> OPEN_LOCAL_IMAGE * * * * * * * * * * * * *
    *    |         |   .   ^                             *
@@ -167,6 +170,7 @@ private:
   cls::journal::Client m_client;
   uint64_t m_remote_tag_class = 0;
   ImageCtxT *m_remote_image_ctx = nullptr;
+  bool m_primary = false;
   bool m_created_local_image = false;
   int m_ret_val = 0;
 
@@ -187,6 +191,9 @@ private:
   void open_remote_image();
   void handle_open_remote_image(int r);
 
+  void is_primary();
+  void handle_is_primary(int r);
+
   void update_client_state();
   void handle_update_client_state(int r);
 
diff --git a/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.cc b/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.cc
new file mode 100644
index 0000000000000..a9169f0808a1c
--- /dev/null
+++ b/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.cc
@@ -0,0 +1,74 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "IsPrimaryRequest.h"
+#include "common/errno.h"
+#include "common/WorkQueue.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Journal.h"
+#include "librbd/Utils.h"
+#include <type_traits>
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::image_replayer::IsPrimaryRequest: " \
+                           << this << " " << __func__ << " "
+
+namespace rbd {
+namespace mirror {
+namespace image_replayer {
+
+using librbd::util::create_context_callback;
+
+template <typename I>
+IsPrimaryRequest<I>::IsPrimaryRequest(I *image_ctx, bool *primary,
+                                      Context *on_finish)
+  : m_image_ctx(image_ctx), m_primary(primary), m_on_finish(on_finish) {
+}
+
+template <typename I>
+void IsPrimaryRequest<I>::send() {
+  send_is_tag_owner();
+}
+
+template <typename I>
+void IsPrimaryRequest<I>::send_is_tag_owner() {
+  // deduce the class type for the journal to support unit tests
+  using Journal = typename std::decay<
+    typename std::remove_pointer<decltype(std::declval<I>().journal)>
+    ::type>::type;
+
+  dout(20) << dendl;
+
+  Context *ctx = create_context_callback<
+    IsPrimaryRequest<I>, &IsPrimaryRequest<I>::handle_is_tag_owner>(this);
+
+  Journal::is_tag_owner(m_image_ctx, m_primary, ctx);
+}
+
+template <typename I>
+void IsPrimaryRequest<I>::handle_is_tag_owner(int r) {
+  dout(20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    derr << ": failed to query remote image tag owner: " << cpp_strerror(r)
+         << dendl;
+  }
+
+  finish(r);
+}
+
+template <typename I>
+void IsPrimaryRequest<I>::finish(int r) {
+  dout(20) << ": r=" << r << dendl;
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace image_replayer
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::image_replayer::IsPrimaryRequest<librbd::ImageCtx>;
diff --git a/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.h b/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.h
new file mode 100644
index 0000000000000..e3482084b8a27
--- /dev/null
+++ b/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.h
@@ -0,0 +1,57 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_IMAGE_REPLAYER_IS_PRIMARY_REQUEST_H
+#define RBD_MIRROR_IMAGE_REPLAYER_IS_PRIMARY_REQUEST_H
+
+class Context;
+class ContextWQ;
+namespace librbd { class ImageCtx; }
+
+namespace rbd {
+namespace mirror {
+namespace image_replayer {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class IsPrimaryRequest {
+public:
+  static IsPrimaryRequest* create(ImageCtxT *image_ctx, bool *primary,
+                                  Context *on_finish) {
+    return new IsPrimaryRequest(image_ctx, primary, on_finish);
+  }
+
+  IsPrimaryRequest(ImageCtxT *image_ctx, bool *primary, Context *on_finish);
+
+  void send();
+
+private:
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   * IS_TAG_OWNER * * * * * * *
+   *    |                     * (error)
+   *    v                     *
+   * <finish> < * * * * * * * *
+   *
+   * @endverbatim
+   */
+  ImageCtxT *m_image_ctx;
+  bool *m_primary;
+  Context *m_on_finish;
+
+  void send_is_tag_owner();
+  void handle_is_tag_owner(int r);
+
+  void finish(int r);
+};
+
+} // namespace image_replayer
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::image_replayer::IsPrimaryRequest<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_IMAGE_REPLAYER_IS_PRIMARY_REQUEST_H
diff --git a/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc b/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc
index 49e3082d005a3..a17e8b7df4072 100644
--- a/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc
+++ b/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc
@@ -2,8 +2,9 @@
 // vim: ts=8 sw=2 smarttab
 
 #include "include/compat.h"
-#include "OpenLocalImageRequest.h"
 #include "CloseImageRequest.h"
+#include "IsPrimaryRequest.h"
+#include "OpenLocalImageRequest.h"
 #include "common/errno.h"
 #include "common/WorkQueue.h"
 #include "librbd/ExclusiveLock.h"
@@ -114,42 +115,54 @@ void OpenLocalImageRequest<I>::handle_open_image(int r) {
     return;
   }
 
-  send_lock_image();
+  send_is_primary();
 }
 
 template <typename I>
-void OpenLocalImageRequest<I>::send_lock_image() {
-  // deduce the class type for the journal to support unit tests
-  using Journal = typename std::decay<
-    typename std::remove_pointer<decltype(std::declval<I>().journal)>
-    ::type>::type;
-
+void OpenLocalImageRequest<I>::send_is_primary() {
   dout(20) << dendl;
 
-  RWLock::RLocker owner_locker((*m_local_image_ctx)->owner_lock);
-  if ((*m_local_image_ctx)->exclusive_lock == nullptr) {
-    derr << ": image does not support exclusive lock" << dendl;
-    send_close_image(false, -EINVAL);
-    return;
-  }
+  Context *ctx = create_context_callback<
+    OpenLocalImageRequest<I>, &OpenLocalImageRequest<I>::handle_is_primary>(
+      this);
+  IsPrimaryRequest<I> *request = IsPrimaryRequest<I>::create(*m_local_image_ctx,
+                                                             &m_primary, ctx);
+  request->send();
+}
+
+template <typename I>
+void OpenLocalImageRequest<I>::handle_is_primary(int r) {
+  dout(20) << ": r=" << r << dendl;
 
-  // TODO: make an async version
-  bool tag_owner;
-  int r = Journal::is_tag_owner(*m_local_image_ctx, &tag_owner);
   if (r < 0) {
-    derr << ": failed to query journal: " << cpp_strerror(r) << dendl;
+    derr << ": error querying local image primary status: " << cpp_strerror(r)
+         << dendl;
     send_close_image(false, r);
     return;
   }
 
   // if the local image owns the tag -- don't steal the lock since
   // we aren't going to mirror peer data into this image anyway
-  if (tag_owner) {
+  if (m_primary) {
     dout(10) << ": local image is primary -- skipping image replay" << dendl;
     send_close_image(false, -EREMOTEIO);
     return;
   }
 
+  send_lock_image();
+}
+
+template <typename I>
+void OpenLocalImageRequest<I>::send_lock_image() {
+  dout(20) << dendl;
+
+  RWLock::RLocker owner_locker((*m_local_image_ctx)->owner_lock);
+  if ((*m_local_image_ctx)->exclusive_lock == nullptr) {
+    derr << ": image does not support exclusive lock" << dendl;
+    send_close_image(false, -EINVAL);
+    return;
+  }
+
   // disallow any proxied maintenance operations before grabbing lock
   (*m_local_image_ctx)->exclusive_lock->block_requests(-EROFS);
 
diff --git a/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h b/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h
index e40b1c2a06226..2a1bbb2a8e2e8 100644
--- a/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h
+++ b/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h
@@ -48,6 +48,9 @@ private:
    *    v
    * OPEN_IMAGE * * * * * * * *
    *    |                     *
+   *    v                     *
+   * IS_PRIMARY * * * * * * * *
+   *    |                     *
    *    v (skip if primary)   v
    * LOCK_IMAGE * * * > CLOSE_IMAGE
    *    |                     |
@@ -63,11 +66,15 @@ private:
   ContextWQ *m_work_queue;
   Context *m_on_finish;
 
+  bool m_primary = false;
   int m_ret_val = 0;
 
   void send_open_image();
   void handle_open_image(int r);
 
+  void send_is_primary();
+  void handle_is_primary(int r);
+
   void send_lock_image();
   void handle_lock_image(int r);