rbd-mirror: improve detection of blacklisted state

Fixes: https://tracker.ceph.com/issues/44159
Signed-off-by: Mykola Golub <mgolub@suse.com>
(cherry picked from commit cfb4f423a42d0265cb78ebb4eb8cc6924d6f45fa)

Conflicts:
	src/tools/rbd_mirror/InstanceReplayer.h (ceph::mutex vs Mutex)
	src/tools/rbd_mirror/NamespaceReplayer.cc (does not exist)
	src/tools/rbd_mirror/PoolReplayer.cc (code from NamespaceReplayer is here)
	src/test/rbd_mirror/test_mock_PoolReplayer.cc (accordingly to PoolReplayer.cc changes)

diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc
index 4ceb9be3c4762ca2e6991e65b1ca7e9fb84acced..e6bc9b1e2f5f8ed73c5c47f8b393ec652c120578 100644 (file)
--- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc
@@ -116,6 +116,8 @@ struct InstanceReplayer<librbd::MockTestImageCtx> {
     return s_instance;
   }
 
+  MOCK_METHOD0(is_blacklisted, bool());
+
   MOCK_METHOD0(start, void());
   MOCK_METHOD0(stop, void());
   MOCK_METHOD0(restart, void());
@@ -189,6 +191,7 @@ struct LeaderWatcher<librbd::MockTestImageCtx> {
     return s_instance;
   }
 
+  MOCK_METHOD0(is_blacklisted, bool());
   MOCK_METHOD0(is_leader, bool());
   MOCK_METHOD0(release_leader, void());
 
@@ -330,6 +333,12 @@ public:
                       Return(r)));
   }
 
+  void expect_instance_replayer_is_blacklisted(
+      MockInstanceReplayer &mock_instance_replayer, bool blacklisted) {
+    EXPECT_CALL(mock_instance_replayer, is_blacklisted())
+      .WillRepeatedly(Return(blacklisted));
+  }
+
   void expect_instance_replayer_init(MockInstanceReplayer& mock_instance_replayer) {
     EXPECT_CALL(mock_instance_replayer, init());
   }
@@ -364,6 +373,12 @@ public:
     EXPECT_CALL(mock_instance_watcher, shut_down());
   }
 
+  void expect_leader_watcher_is_blacklisted(
+      MockLeaderWatcher &mock_leader_watcher, bool blacklisted) {
+    EXPECT_CALL(mock_leader_watcher, is_blacklisted())
+      .WillRepeatedly(Return(blacklisted));
+  }
+
   void expect_leader_watcher_init(MockLeaderWatcher& mock_leader_watcher,
                                   int r) {
     EXPECT_CALL(mock_leader_watcher, init())
@@ -394,6 +409,12 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) {
   peer_spec.mon_host = "123";
   peer_spec.key = "234";
 
+  auto mock_instance_replayer = new MockInstanceReplayer();
+  expect_instance_replayer_is_blacklisted(*mock_instance_replayer, false);
+
+  auto mock_leader_watcher = new MockLeaderWatcher();
+  expect_leader_watcher_is_blacklisted(*mock_leader_watcher, false);
+
   InSequence seq;
 
   auto& mock_cluster = get_mock_cluster();
@@ -413,7 +434,6 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) {
 
   expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
 
-  auto mock_instance_replayer = new MockInstanceReplayer();
   expect_instance_replayer_init(*mock_instance_replayer);
   expect_instance_replayer_add_peer(*mock_instance_replayer, "uuid");
 
@@ -424,7 +444,6 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) {
   expect_service_daemon_add_or_update_instance_id_attribute(
       *mock_instance_watcher, mock_service_daemon);
 
-  auto mock_leader_watcher = new MockLeaderWatcher();
   expect_leader_watcher_init(*mock_leader_watcher, 0);
 
   MockThreads mock_threads(m_threads);

diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc
index 6d6b03ea8dbc34f8e703083d90072dd561246f3a..c0086a48f5b7dc122d99c716aeec4707b9af5493 100644 (file)
--- a/src/tools/rbd_mirror/InstanceReplayer.cc
+++ b/src/tools/rbd_mirror/InstanceReplayer.cc
@@ -49,6 +49,12 @@ InstanceReplayer<I>::~InstanceReplayer() {
   ceph_assert(m_image_replayers.empty());
 }
 
+template <typename I>
+bool InstanceReplayer<I>::is_blacklisted() const {
+  std::lock_guard locker{m_lock};
+  return m_blacklisted;
+}
+
 template <typename I>
 int InstanceReplayer<I>::init() {
   C_SaferCond init_ctx;
@@ -303,6 +309,7 @@ void InstanceReplayer<I>::start_image_replayer(
   } else if (image_replayer->is_blacklisted()) {
     derr << "global_image_id=" << global_image_id << ": blacklisted detected "
          << "during image replay" << dendl;
+    m_blacklisted = true;
     return;
   } else if (image_replayer->is_finished()) {
     // TODO temporary until policy integrated

diff --git a/src/tools/rbd_mirror/InstanceReplayer.h b/src/tools/rbd_mirror/InstanceReplayer.h
index 0321d78c5afb34faa872a8487569835446ddeee7..efbdde02d175f5e98dc3a6f67ece4e7e58ecd681 100644 (file)
--- a/src/tools/rbd_mirror/InstanceReplayer.h
+++ b/src/tools/rbd_mirror/InstanceReplayer.h
@@ -43,6 +43,8 @@ public:
                   int64_t local_pool_id);
   ~InstanceReplayer();
 
+  bool is_blacklisted() const;
+
   int init();
   void shut_down();
 
@@ -87,13 +89,14 @@ private:
   std::string m_local_mirror_uuid;
   int64_t m_local_pool_id;
 
-  Mutex m_lock;
+  mutable Mutex m_lock;
   AsyncOpTracker m_async_op_tracker;
   std::map<std::string, ImageReplayer<ImageCtxT> *> m_image_replayers;
   Peers m_peers;
   Context *m_image_state_check_task = nullptr;
   Context *m_on_shut_down = nullptr;
   bool m_manual_stop = false;
+  bool m_blacklisted = false;
 
   void wait_for_ops();
   void handle_wait_for_ops(int r);

diff --git a/src/tools/rbd_mirror/LeaderWatcher.cc b/src/tools/rbd_mirror/LeaderWatcher.cc
index f3291bc60962a242ce5ddd4c3f6aa431eefe8b20..0d4bde6f33947bbd7a06217dae71dd26341580f0 100644 (file)
--- a/src/tools/rbd_mirror/LeaderWatcher.cc
+++ b/src/tools/rbd_mirror/LeaderWatcher.cc
@@ -246,6 +246,12 @@ void LeaderWatcher<I>::handle_wait_for_tasks() {
   m_work_queue->queue(ctx, 0);
 }
 
+template <typename I>
+bool LeaderWatcher<I>::is_blacklisted() const {
+  std::lock_guard locker{m_lock};
+  return m_blacklisted;
+}
+
 template <typename I>
 bool LeaderWatcher<I>::is_leader() const {
   Mutex::Locker locker(m_lock);
@@ -1080,7 +1086,7 @@ void LeaderWatcher<I>::handle_notify(uint64_t notify_id, uint64_t handle,
     auto iter = bl.cbegin();
     decode(notify_message, iter);
   } catch (const buffer::error &err) {
-    derr << ": error decoding image notification: " << err.what() << dendl;
+    derr << "error decoding image notification: " << err.what() << dendl;
     ctx->complete(0);
     return;
   }
@@ -1092,9 +1098,13 @@ template <typename I>
 void LeaderWatcher<I>::handle_rewatch_complete(int r) {
   dout(5) << "r=" << r << dendl;
 
-  if (r != -EBLACKLISTED) {
-    m_leader_lock->reacquire_lock(nullptr);
+  if (r == -EBLACKLISTED) {
+    dout(1) << "blacklisted detected" << dendl;
+    m_blacklisted = true;
+    return;
   }
+
+  m_leader_lock->reacquire_lock(nullptr);
 }
 
 template <typename I>

diff --git a/src/tools/rbd_mirror/LeaderWatcher.h b/src/tools/rbd_mirror/LeaderWatcher.h
index 54ee5dc82c3290c6fc99f0fbed05b443513f7db8..01ee0565d4cd898f510ac7d66b0f20519ba83311 100644 (file)
--- a/src/tools/rbd_mirror/LeaderWatcher.h
+++ b/src/tools/rbd_mirror/LeaderWatcher.h
@@ -45,6 +45,7 @@ public:
   void init(Context *on_finish);
   void shut_down(Context *on_finish);
 
+  bool is_blacklisted() const;
   bool is_leader() const;
   bool is_releasing_leader() const;
   bool get_leader_instance_id(std::string *instance_id) const;
@@ -220,6 +221,8 @@ private:
   Instances<ImageCtxT> *m_instances = nullptr;
   librbd::managed_lock::Locker m_locker;
 
+  bool m_blacklisted = false;
+
   AsyncOpTracker m_timer_op_tracker;
   Context *m_timer_task = nullptr;
   C_TimerGate *m_timer_gate = nullptr;

diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc
index ac3463256704a9f95fa67ac953b53fb9a5823f8d..94e91f5cc9be77ffc3336995ecbf2b3eba3e1ce7 100644 (file)
--- a/src/tools/rbd_mirror/PoolReplayer.cc
+++ b/src/tools/rbd_mirror/PoolReplayer.cc
@@ -551,7 +551,9 @@ void PoolReplayer<I>::run()
     }
 
     Mutex::Locker locker(m_lock);
-    if ((m_local_pool_watcher && m_local_pool_watcher->is_blacklisted()) ||
+    if (m_leader_watcher->is_blacklisted() ||
+        m_instance_replayer->is_blacklisted() ||
+        (m_local_pool_watcher && m_local_pool_watcher->is_blacklisted()) ||
        (m_remote_pool_watcher && m_remote_pool_watcher->is_blacklisted())) {
       m_blacklisted = true;
       m_stopping = true;