From cfb4f423a42d0265cb78ebb4eb8cc6924d6f45fa Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Wed, 19 Feb 2020 10:17:08 +0000 Subject: [PATCH] rbd-mirror: improve detection of blacklisted state Fixes: https://tracker.ceph.com/issues/44159 Signed-off-by: Mykola Golub --- src/test/rbd_mirror/test_mock_PoolReplayer.cc | 11 +++++++++++ src/tools/rbd_mirror/InstanceReplayer.cc | 7 +++++++ src/tools/rbd_mirror/InstanceReplayer.h | 5 ++++- src/tools/rbd_mirror/LeaderWatcher.cc | 16 +++++++++++++--- src/tools/rbd_mirror/LeaderWatcher.h | 3 +++ src/tools/rbd_mirror/NamespaceReplayer.cc | 3 ++- src/tools/rbd_mirror/PoolReplayer.cc | 3 ++- 7 files changed, 42 insertions(+), 6 deletions(-) diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc index e8976ef24f7..42e5a42a6ea 100644 --- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc +++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc @@ -186,6 +186,7 @@ struct LeaderWatcher { return s_instance; } + MOCK_METHOD0(is_blacklisted, bool()); MOCK_METHOD0(is_leader, bool()); MOCK_METHOD0(release_leader, void()); @@ -409,6 +410,12 @@ public: })); } + void expect_leader_watcher_is_blacklisted( + MockLeaderWatcher &mock_leader_watcher, bool blacklisted) { + EXPECT_CALL(mock_leader_watcher, is_blacklisted()) + .WillRepeatedly(Return(blacklisted)); + } + void expect_namespace_replayer_is_blacklisted( MockNamespaceReplayer &mock_namespace_replayer, bool blacklisted) { @@ -543,6 +550,7 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { auto mock_leader_watcher = new MockLeaderWatcher(); expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher); + expect_leader_watcher_is_blacklisted(*mock_leader_watcher, false); InSequence seq; @@ -605,6 +613,7 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) { auto mock_leader_watcher = new MockLeaderWatcher(); expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher); expect_leader_watcher_list_instances(*mock_leader_watcher); + expect_leader_watcher_is_blacklisted(*mock_leader_watcher, false); InSequence seq; @@ -692,6 +701,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) { auto mock_leader_watcher = new MockLeaderWatcher(); expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher); expect_leader_watcher_list_instances(*mock_leader_watcher); + expect_leader_watcher_is_blacklisted(*mock_leader_watcher, false); auto& mock_cluster = get_mock_cluster(); auto mock_local_rados_client = mock_cluster.do_create_rados_client( @@ -809,6 +819,7 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { auto mock_leader_watcher = new MockLeaderWatcher(); expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher); expect_leader_watcher_list_instances(*mock_leader_watcher); + expect_leader_watcher_is_blacklisted(*mock_leader_watcher, false); auto& mock_cluster = get_mock_cluster(); auto mock_local_rados_client = mock_cluster.do_create_rados_client( diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc index 2b9b47a0e01..9a8b50596c6 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.cc +++ b/src/tools/rbd_mirror/InstanceReplayer.cc @@ -56,6 +56,12 @@ InstanceReplayer::~InstanceReplayer() { ceph_assert(m_image_replayers.empty()); } +template +bool InstanceReplayer::is_blacklisted() const { + std::lock_guard locker{m_lock}; + return m_blacklisted; +} + template int InstanceReplayer::init() { C_SaferCond init_ctx; @@ -327,6 +333,7 @@ void InstanceReplayer::start_image_replayer( } else if (image_replayer->is_blacklisted()) { derr << "global_image_id=" << global_image_id << ": blacklisted detected " << "during image replay" << dendl; + m_blacklisted = true; return; } else if (image_replayer->is_finished()) { // TODO temporary until policy integrated diff --git a/src/tools/rbd_mirror/InstanceReplayer.h b/src/tools/rbd_mirror/InstanceReplayer.h index 3119fd3ebda..ec500fd5cb3 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.h +++ b/src/tools/rbd_mirror/InstanceReplayer.h @@ -52,6 +52,8 @@ public: PoolMetaCache* pool_meta_cache); ~InstanceReplayer(); + bool is_blacklisted() const; + int init(); void shut_down(); @@ -102,13 +104,14 @@ private: journal::CacheManagerHandler *m_cache_manager_handler; PoolMetaCache* m_pool_meta_cache; - ceph::mutex m_lock; + mutable ceph::mutex m_lock; AsyncOpTracker m_async_op_tracker; std::map *> m_image_replayers; Peers m_peers; Context *m_image_state_check_task = nullptr; Context *m_on_shut_down = nullptr; bool m_manual_stop = false; + bool m_blacklisted = false; void wait_for_ops(); void handle_wait_for_ops(int r); diff --git a/src/tools/rbd_mirror/LeaderWatcher.cc b/src/tools/rbd_mirror/LeaderWatcher.cc index 01795553560..1581319219d 100644 --- a/src/tools/rbd_mirror/LeaderWatcher.cc +++ b/src/tools/rbd_mirror/LeaderWatcher.cc @@ -248,6 +248,12 @@ void LeaderWatcher::handle_wait_for_tasks() { m_work_queue->queue(ctx, 0); } +template +bool LeaderWatcher::is_blacklisted() const { + std::lock_guard locker{m_lock}; + return m_blacklisted; +} + template bool LeaderWatcher::is_leader() const { std::lock_guard locker{m_lock}; @@ -1003,7 +1009,7 @@ void LeaderWatcher::handle_notify(uint64_t notify_id, uint64_t handle, auto iter = bl.cbegin(); decode(notify_message, iter); } catch (const buffer::error &err) { - derr << ": error decoding image notification: " << err.what() << dendl; + derr << "error decoding image notification: " << err.what() << dendl; ctx->complete(0); return; } @@ -1015,9 +1021,13 @@ template void LeaderWatcher::handle_rewatch_complete(int r) { dout(5) << "r=" << r << dendl; - if (r != -EBLACKLISTED) { - m_leader_lock->reacquire_lock(nullptr); + if (r == -EBLACKLISTED) { + dout(1) << "blacklisted detected" << dendl; + m_blacklisted = true; + return; } + + m_leader_lock->reacquire_lock(nullptr); } template diff --git a/src/tools/rbd_mirror/LeaderWatcher.h b/src/tools/rbd_mirror/LeaderWatcher.h index 67c971dbcfb..f4573a5b1d9 100644 --- a/src/tools/rbd_mirror/LeaderWatcher.h +++ b/src/tools/rbd_mirror/LeaderWatcher.h @@ -44,6 +44,7 @@ public: void init(Context *on_finish); void shut_down(Context *on_finish); + bool is_blacklisted() const; bool is_leader() const; bool is_releasing_leader() const; bool get_leader_instance_id(std::string *instance_id) const; @@ -215,6 +216,8 @@ private: Instances *m_instances = nullptr; librbd::managed_lock::Locker m_locker; + bool m_blacklisted = false; + AsyncOpTracker m_timer_op_tracker; Context *m_timer_task = nullptr; C_TimerGate *m_timer_gate = nullptr; diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc index 582b875835e..e0c593f99d8 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.cc +++ b/src/tools/rbd_mirror/NamespaceReplayer.cc @@ -72,7 +72,8 @@ NamespaceReplayer::NamespaceReplayer( template bool NamespaceReplayer::is_blacklisted() const { std::lock_guard locker{m_lock}; - return (m_local_pool_watcher && + return m_instance_replayer->is_blacklisted() || + (m_local_pool_watcher && m_local_pool_watcher->is_blacklisted()) || (m_remote_pool_watcher && m_remote_pool_watcher->is_blacklisted()); diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index 6c8f1c47714..75067813aa5 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -590,7 +590,8 @@ void PoolReplayer::run() { std::unique_lock locker{m_lock}; - if (m_default_namespace_replayer->is_blacklisted()) { + if (m_leader_watcher->is_blacklisted() || + m_default_namespace_replayer->is_blacklisted()) { m_blacklisted = true; m_stopping = true; } -- 2.39.5