From: Mykola Golub Date: Wed, 19 Feb 2020 10:17:08 +0000 (+0000) Subject: rbd-mirror: improve detection of blacklisted state X-Git-Tag: v14.2.10~218^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F33533%2Fhead;p=ceph.git rbd-mirror: improve detection of blacklisted state Fixes: https://tracker.ceph.com/issues/44159 Signed-off-by: Mykola Golub (cherry picked from commit cfb4f423a42d0265cb78ebb4eb8cc6924d6f45fa) Conflicts: src/tools/rbd_mirror/InstanceReplayer.h (ceph::mutex vs Mutex) src/tools/rbd_mirror/NamespaceReplayer.cc (does not exist) src/tools/rbd_mirror/PoolReplayer.cc (code from NamespaceReplayer is here) src/test/rbd_mirror/test_mock_PoolReplayer.cc (accordingly to PoolReplayer.cc changes) --- diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc index 4ceb9be3c476..e6bc9b1e2f5f 100644 --- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc +++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc @@ -116,6 +116,8 @@ struct InstanceReplayer { return s_instance; } + MOCK_METHOD0(is_blacklisted, bool()); + MOCK_METHOD0(start, void()); MOCK_METHOD0(stop, void()); MOCK_METHOD0(restart, void()); @@ -189,6 +191,7 @@ struct LeaderWatcher { return s_instance; } + MOCK_METHOD0(is_blacklisted, bool()); MOCK_METHOD0(is_leader, bool()); MOCK_METHOD0(release_leader, void()); @@ -330,6 +333,12 @@ public: Return(r))); } + void expect_instance_replayer_is_blacklisted( + MockInstanceReplayer &mock_instance_replayer, bool blacklisted) { + EXPECT_CALL(mock_instance_replayer, is_blacklisted()) + .WillRepeatedly(Return(blacklisted)); + } + void expect_instance_replayer_init(MockInstanceReplayer& mock_instance_replayer) { EXPECT_CALL(mock_instance_replayer, init()); } @@ -364,6 +373,12 @@ public: EXPECT_CALL(mock_instance_watcher, shut_down()); } + void expect_leader_watcher_is_blacklisted( + MockLeaderWatcher &mock_leader_watcher, bool blacklisted) { + EXPECT_CALL(mock_leader_watcher, is_blacklisted()) + .WillRepeatedly(Return(blacklisted)); + } + void expect_leader_watcher_init(MockLeaderWatcher& mock_leader_watcher, int r) { EXPECT_CALL(mock_leader_watcher, init()) @@ -394,6 +409,12 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { peer_spec.mon_host = "123"; peer_spec.key = "234"; + auto mock_instance_replayer = new MockInstanceReplayer(); + expect_instance_replayer_is_blacklisted(*mock_instance_replayer, false); + + auto mock_leader_watcher = new MockLeaderWatcher(); + expect_leader_watcher_is_blacklisted(*mock_leader_watcher, false); + InSequence seq; auto& mock_cluster = get_mock_cluster(); @@ -413,7 +434,6 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0); - auto mock_instance_replayer = new MockInstanceReplayer(); expect_instance_replayer_init(*mock_instance_replayer); expect_instance_replayer_add_peer(*mock_instance_replayer, "uuid"); @@ -424,7 +444,6 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { expect_service_daemon_add_or_update_instance_id_attribute( *mock_instance_watcher, mock_service_daemon); - auto mock_leader_watcher = new MockLeaderWatcher(); expect_leader_watcher_init(*mock_leader_watcher, 0); MockThreads mock_threads(m_threads); diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc index 6d6b03ea8dbc..c0086a48f5b7 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.cc +++ b/src/tools/rbd_mirror/InstanceReplayer.cc @@ -49,6 +49,12 @@ InstanceReplayer::~InstanceReplayer() { ceph_assert(m_image_replayers.empty()); } +template +bool InstanceReplayer::is_blacklisted() const { + std::lock_guard locker{m_lock}; + return m_blacklisted; +} + template int InstanceReplayer::init() { C_SaferCond init_ctx; @@ -303,6 +309,7 @@ void InstanceReplayer::start_image_replayer( } else if (image_replayer->is_blacklisted()) { derr << "global_image_id=" << global_image_id << ": blacklisted detected " << "during image replay" << dendl; + m_blacklisted = true; return; } else if (image_replayer->is_finished()) { // TODO temporary until policy integrated diff --git a/src/tools/rbd_mirror/InstanceReplayer.h b/src/tools/rbd_mirror/InstanceReplayer.h index 0321d78c5afb..efbdde02d175 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.h +++ b/src/tools/rbd_mirror/InstanceReplayer.h @@ -43,6 +43,8 @@ public: int64_t local_pool_id); ~InstanceReplayer(); + bool is_blacklisted() const; + int init(); void shut_down(); @@ -87,13 +89,14 @@ private: std::string m_local_mirror_uuid; int64_t m_local_pool_id; - Mutex m_lock; + mutable Mutex m_lock; AsyncOpTracker m_async_op_tracker; std::map *> m_image_replayers; Peers m_peers; Context *m_image_state_check_task = nullptr; Context *m_on_shut_down = nullptr; bool m_manual_stop = false; + bool m_blacklisted = false; void wait_for_ops(); void handle_wait_for_ops(int r); diff --git a/src/tools/rbd_mirror/LeaderWatcher.cc b/src/tools/rbd_mirror/LeaderWatcher.cc index f3291bc60962..0d4bde6f3394 100644 --- a/src/tools/rbd_mirror/LeaderWatcher.cc +++ b/src/tools/rbd_mirror/LeaderWatcher.cc @@ -246,6 +246,12 @@ void LeaderWatcher::handle_wait_for_tasks() { m_work_queue->queue(ctx, 0); } +template +bool LeaderWatcher::is_blacklisted() const { + std::lock_guard locker{m_lock}; + return m_blacklisted; +} + template bool LeaderWatcher::is_leader() const { Mutex::Locker locker(m_lock); @@ -1080,7 +1086,7 @@ void LeaderWatcher::handle_notify(uint64_t notify_id, uint64_t handle, auto iter = bl.cbegin(); decode(notify_message, iter); } catch (const buffer::error &err) { - derr << ": error decoding image notification: " << err.what() << dendl; + derr << "error decoding image notification: " << err.what() << dendl; ctx->complete(0); return; } @@ -1092,9 +1098,13 @@ template void LeaderWatcher::handle_rewatch_complete(int r) { dout(5) << "r=" << r << dendl; - if (r != -EBLACKLISTED) { - m_leader_lock->reacquire_lock(nullptr); + if (r == -EBLACKLISTED) { + dout(1) << "blacklisted detected" << dendl; + m_blacklisted = true; + return; } + + m_leader_lock->reacquire_lock(nullptr); } template diff --git a/src/tools/rbd_mirror/LeaderWatcher.h b/src/tools/rbd_mirror/LeaderWatcher.h index 54ee5dc82c32..01ee0565d4cd 100644 --- a/src/tools/rbd_mirror/LeaderWatcher.h +++ b/src/tools/rbd_mirror/LeaderWatcher.h @@ -45,6 +45,7 @@ public: void init(Context *on_finish); void shut_down(Context *on_finish); + bool is_blacklisted() const; bool is_leader() const; bool is_releasing_leader() const; bool get_leader_instance_id(std::string *instance_id) const; @@ -220,6 +221,8 @@ private: Instances *m_instances = nullptr; librbd::managed_lock::Locker m_locker; + bool m_blacklisted = false; + AsyncOpTracker m_timer_op_tracker; Context *m_timer_task = nullptr; C_TimerGate *m_timer_gate = nullptr; diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index ac3463256704..94e91f5cc9be 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -551,7 +551,9 @@ void PoolReplayer::run() } Mutex::Locker locker(m_lock); - if ((m_local_pool_watcher && m_local_pool_watcher->is_blacklisted()) || + if (m_leader_watcher->is_blacklisted() || + m_instance_replayer->is_blacklisted() || + (m_local_pool_watcher && m_local_pool_watcher->is_blacklisted()) || (m_remote_pool_watcher && m_remote_pool_watcher->is_blacklisted())) { m_blacklisted = true; m_stopping = true;