From 7dc99b2591b91bafd01f6697c5019e2c23ba671c Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Wed, 19 Feb 2020 10:17:08 +0000 Subject: [PATCH] rbd-mirror: improve detection of blacklisted state Fixes: https://tracker.ceph.com/issues/44159 Signed-off-by: Mykola Golub (cherry picked from commit cfb4f423a42d0265cb78ebb4eb8cc6924d6f45fa) Conflicts: src/tools/rbd_mirror/InstanceReplayer.h (ceph::mutex vs Mutex) src/tools/rbd_mirror/InstanceReplayer.cc (Mutex::Locker vs std::lock_guard) src/tools/rbd_mirror/LeaderWatcher.cc (Mutex::Locker vs std::lock_guard) src/tools/rbd_mirror/NamespaceReplayer.cc (does not exist) src/tools/rbd_mirror/PoolReplayer.cc (code from NamespaceReplayer is here) src/test/rbd_mirror/test_mock_PoolReplayer.cc (does not exist) --- src/tools/rbd_mirror/InstanceReplayer.cc | 7 +++++++ src/tools/rbd_mirror/InstanceReplayer.h | 5 ++++- src/tools/rbd_mirror/LeaderWatcher.cc | 17 ++++++++++++++--- src/tools/rbd_mirror/LeaderWatcher.h | 3 +++ src/tools/rbd_mirror/PoolReplayer.cc | 4 +++- 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc index 0e21bf5f97a35..c6401e5a5feaa 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.cc +++ b/src/tools/rbd_mirror/InstanceReplayer.cc @@ -49,6 +49,12 @@ InstanceReplayer::~InstanceReplayer() { assert(m_image_replayers.empty()); } +template +bool InstanceReplayer::is_blacklisted() const { + Mutex::Locker locker(m_lock); + return m_blacklisted; +} + template int InstanceReplayer::init() { C_SaferCond init_ctx; @@ -303,6 +309,7 @@ void InstanceReplayer::start_image_replayer( } else if (image_replayer->is_blacklisted()) { derr << "global_image_id=" << global_image_id << ": blacklisted detected " << "during image replay" << dendl; + m_blacklisted = true; return; } else if (image_replayer->is_finished()) { // TODO temporary until policy integrated diff --git a/src/tools/rbd_mirror/InstanceReplayer.h b/src/tools/rbd_mirror/InstanceReplayer.h index 96969cb62c9be..c6b936a9d6cf1 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.h +++ b/src/tools/rbd_mirror/InstanceReplayer.h @@ -43,6 +43,8 @@ public: int64_t local_pool_id); ~InstanceReplayer(); + bool is_blacklisted() const; + int init(); void shut_down(); @@ -87,13 +89,14 @@ private: std::string m_local_mirror_uuid; int64_t m_local_pool_id; - Mutex m_lock; + mutable Mutex m_lock; AsyncOpTracker m_async_op_tracker; std::map *> m_image_replayers; Peers m_peers; Context *m_image_state_check_task = nullptr; Context *m_on_shut_down = nullptr; bool m_manual_stop = false; + bool m_blacklisted = false; void wait_for_ops(); void handle_wait_for_ops(int r); diff --git a/src/tools/rbd_mirror/LeaderWatcher.cc b/src/tools/rbd_mirror/LeaderWatcher.cc index 861c6073074d7..8146ec9a04b1d 100644 --- a/src/tools/rbd_mirror/LeaderWatcher.cc +++ b/src/tools/rbd_mirror/LeaderWatcher.cc @@ -246,6 +246,13 @@ void LeaderWatcher::handle_wait_for_tasks() { m_work_queue->queue(ctx, 0); } +template +bool LeaderWatcher::is_blacklisted() const { + Mutex::Locker locker(m_lock); + + return m_blacklisted; +} + template bool LeaderWatcher::is_leader() const { Mutex::Locker locker(m_lock); @@ -1080,7 +1087,7 @@ void LeaderWatcher::handle_notify(uint64_t notify_id, uint64_t handle, bufferlist::iterator iter = bl.begin(); decode(notify_message, iter); } catch (const buffer::error &err) { - derr << ": error decoding image notification: " << err.what() << dendl; + derr << "error decoding image notification: " << err.what() << dendl; ctx->complete(0); return; } @@ -1092,9 +1099,13 @@ template void LeaderWatcher::handle_rewatch_complete(int r) { dout(5) << "r=" << r << dendl; - if (r != -EBLACKLISTED) { - m_leader_lock->reacquire_lock(nullptr); + if (r == -EBLACKLISTED) { + dout(1) << "blacklisted detected" << dendl; + m_blacklisted = true; + return; } + + m_leader_lock->reacquire_lock(nullptr); } template diff --git a/src/tools/rbd_mirror/LeaderWatcher.h b/src/tools/rbd_mirror/LeaderWatcher.h index 8b37db0ee0661..7e9846ee73200 100644 --- a/src/tools/rbd_mirror/LeaderWatcher.h +++ b/src/tools/rbd_mirror/LeaderWatcher.h @@ -45,6 +45,7 @@ public: void init(Context *on_finish); void shut_down(Context *on_finish); + bool is_blacklisted() const; bool is_leader() const; bool is_releasing_leader() const; bool get_leader_instance_id(std::string *instance_id) const; @@ -220,6 +221,8 @@ private: Instances *m_instances = nullptr; librbd::managed_lock::Locker m_locker; + bool m_blacklisted = false; + AsyncOpTracker m_timer_op_tracker; Context *m_timer_task = nullptr; C_TimerGate *m_timer_gate = nullptr; diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index 995fdc2bda6d1..ef45334409723 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -521,7 +521,9 @@ void PoolReplayer::run() } Mutex::Locker locker(m_lock); - if ((m_local_pool_watcher && m_local_pool_watcher->is_blacklisted()) || + if (m_leader_watcher->is_blacklisted() || + m_instance_replayer->is_blacklisted() || + (m_local_pool_watcher && m_local_pool_watcher->is_blacklisted()) || (m_remote_pool_watcher && m_remote_pool_watcher->is_blacklisted())) { m_blacklisted = true; m_stopping = true; -- 2.39.5