From: Jason Dillaman Date: Wed, 25 Sep 2019 21:11:30 +0000 (-0400) Subject: rbd-mirror: pass remote mirror status updater through to image replayer X-Git-Tag: v15.1.0~1245^2~4 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8ee55076bf0cf91fbf5ed02913893779582655d8;p=ceph.git rbd-mirror: pass remote mirror status updater through to image replayer The image replayer now sends status updates to both the local and remote cluster. Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index 8bf1724f6da..8071caac5d5 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -162,7 +162,7 @@ public: m_global_image_id, m_threads.get(), m_instance_watcher, m_local_status_updater, nullptr); - m_replayer->add_peer("peer uuid", m_remote_ioctx); + m_replayer->add_peer("peer uuid", m_remote_ioctx, nullptr); } void start() diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 9532325faf9..8971af04bc6 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -616,24 +616,30 @@ public: void expect_set_mirror_image_status_repeatedly() { EXPECT_CALL(m_local_status_updater, set_mirror_image_status(_, _, _)) .WillRepeatedly(Invoke([](auto, auto, auto){})); + EXPECT_CALL(m_remote_status_updater, set_mirror_image_status(_, _, _)) + .WillRepeatedly(Invoke([](auto, auto, auto){})); } void expect_mirror_image_status_exists(bool exists) { EXPECT_CALL(m_local_status_updater, exists(_)) .WillOnce(Return(exists)); + EXPECT_CALL(m_remote_status_updater, exists(_)) + .WillOnce(Return(exists)); } void create_image_replayer(MockThreads &mock_threads) { m_image_replayer = new MockImageReplayer( m_local_io_ctx, "local_mirror_uuid", "global image id", &mock_threads, &m_instance_watcher, &m_local_status_updater, nullptr); - m_image_replayer->add_peer("peer_uuid", m_remote_io_ctx); + m_image_replayer->add_peer("peer_uuid", m_remote_io_ctx, + &m_remote_status_updater); } librbd::ImageCtx *m_remote_image_ctx; librbd::ImageCtx *m_local_image_ctx = nullptr; MockInstanceWatcher m_instance_watcher; MockMirrorStatusUpdater m_local_status_updater; + MockMirrorStatusUpdater m_remote_status_updater; MockImageReplayer *m_image_replayer = nullptr; }; @@ -793,13 +799,15 @@ TEST_F(TestMockImageReplayer, PrepareLocalImageError) { MockPrepareLocalImageRequest mock_prepare_local_image_request; MockReplayStatusFormatter mock_replay_status_formatter; - expect_set_mirror_image_status_repeatedly(); + EXPECT_CALL(m_local_status_updater, set_mirror_image_status(_, _, _)) + .WillRepeatedly(Invoke([](auto, auto, auto){})); expect_get_or_send_update(mock_replay_status_formatter); InSequence seq; expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, mock_local_image_ctx.name, "remote mirror uuid", -EINVAL); - expect_mirror_image_status_exists(false); + EXPECT_CALL(m_local_status_updater, exists(_)) + .WillOnce(Return(false)); create_image_replayer(mock_threads); diff --git a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc index 74f105a55d9..0d1b620c53f 100644 --- a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc @@ -92,7 +92,8 @@ struct ImageReplayer { MOCK_METHOD0(restart, void()); MOCK_METHOD0(flush, void()); MOCK_METHOD1(print_status, void(Formatter *)); - MOCK_METHOD2(add_peer, void(const std::string &, librados::IoCtx &)); + MOCK_METHOD3(add_peer, void(const std::string &, librados::IoCtx &, + MirrorStatusUpdater*)); MOCK_METHOD0(get_global_image_id, const std::string &()); MOCK_METHOD0(get_local_image_id, const std::string &()); MOCK_METHOD0(is_running, bool()); @@ -190,12 +191,12 @@ TEST_F(TestMockInstanceReplayer, AcquireReleaseImage) { Context *timer_ctx = nullptr; expect_add_event_after(mock_threads, &timer_ctx); instance_replayer.init(); - instance_replayer.add_peer("peer_uuid", m_remote_io_ctx); + instance_replayer.add_peer("peer_uuid", m_remote_io_ctx, nullptr); // Acquire C_SaferCond on_acquire; - EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _)); + EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _, _)); EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false)); EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false)); @@ -259,12 +260,12 @@ TEST_F(TestMockInstanceReplayer, RemoveFinishedImage) { Context *timer_ctx1 = nullptr; expect_add_event_after(mock_threads, &timer_ctx1); instance_replayer.init(); - instance_replayer.add_peer("peer_uuid", m_remote_io_ctx); + instance_replayer.add_peer("peer_uuid", m_remote_io_ctx, nullptr); // Acquire C_SaferCond on_acquire; - EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _)); + EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _, _)); EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false)); EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false)); @@ -331,11 +332,11 @@ TEST_F(TestMockInstanceReplayer, Reacquire) { Context *timer_ctx = nullptr; expect_add_event_after(mock_threads, &timer_ctx); instance_replayer.init(); - instance_replayer.add_peer("peer_uuid", m_remote_io_ctx); + instance_replayer.add_peer("peer_uuid", m_remote_io_ctx, nullptr); // Acquire - EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _)); + EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _, _)); EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false)); EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false)); diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc index c4daeb85082..6cde475a2f7 100644 --- a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc @@ -113,7 +113,8 @@ struct InstanceReplayer { MOCK_METHOD2(print_status, void(Formatter*, std::stringstream*)); - MOCK_METHOD2(add_peer, void(const std::string&, librados::IoCtx&)); + MOCK_METHOD3(add_peer, void(const std::string&, librados::IoCtx&, + MirrorStatusUpdater*)); MOCK_METHOD1(init, void(Context*)); MOCK_METHOD1(shut_down, void(Context*)); @@ -315,7 +316,7 @@ public: void expect_instance_replayer_add_peer( MockInstanceReplayer& mock_instance_replayer, const std::string& uuid) { - EXPECT_CALL(mock_instance_replayer, add_peer(uuid, _)); + EXPECT_CALL(mock_instance_replayer, add_peer(uuid, _, _)); } void expect_instance_replayer_release_all( diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index a5eb9709e67..b4b1fb649af 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -312,12 +312,16 @@ image_replayer::HealthState ImageReplayer::get_health_state() const { } template -void ImageReplayer::add_peer(const std::string &peer_uuid, - librados::IoCtx &io_ctx) { +void ImageReplayer::add_peer( + const std::string &peer_uuid, librados::IoCtx &io_ctx, + MirrorStatusUpdater* remote_status_updater) { + dout(10) << "peer_uuid=" << &peer_uuid << ", " + << "remote_status_updater=" << remote_status_updater << dendl; + std::lock_guard locker{m_lock}; auto it = m_peers.find({peer_uuid}); if (it == m_peers.end()) { - m_peers.insert({peer_uuid, io_ctx}); + m_peers.insert({peer_uuid, io_ctx, remote_status_updater}); } } @@ -1442,6 +1446,10 @@ void ImageReplayer::set_mirror_image_status_update( dout(15) << "status=" << status << dendl; m_local_status_updater->set_mirror_image_status(m_global_image_id, status, force); + if (m_remote_image.mirror_status_updater != nullptr) { + m_remote_image.mirror_status_updater->set_mirror_image_status( + m_global_image_id, status, force); + } m_in_flight_op_tracker.finish_op(); } @@ -1624,6 +1632,17 @@ void ImageReplayer::handle_shut_down(int r) { return; } + if (m_remote_image.mirror_status_updater != nullptr && + m_remote_image.mirror_status_updater->exists(m_global_image_id)) { + dout(15) << "removing remote mirror image status" << dendl; + auto ctx = new LambdaContext([this, r](int) { + handle_shut_down(r); + }); + m_remote_image.mirror_status_updater->remove_mirror_image_status( + m_global_image_id, ctx); + return; + } + dout(10) << "stop complete" << dendl; ReplayStatusFormatter::destroy(m_replay_status_formatter); m_replay_status_formatter = nullptr; diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index d7aa35bb443..166723b3a0d 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -113,7 +113,8 @@ public: image_replayer::HealthState get_health_state() const; - void add_peer(const std::string &peer_uuid, librados::IoCtx &remote_io_ctx); + void add_peer(const std::string &peer_uuid, librados::IoCtx &remote_io_ctx, + MirrorStatusUpdater* remote_status_updater); inline int64_t get_local_pool_id() const { return m_local_io_ctx.get_id(); @@ -215,6 +216,7 @@ protected: bool on_replay_interrupted(); private: + typedef std::set> Peers; typedef typename librbd::journal::TypeTraits::ReplayEntry ReplayEntry; enum State { @@ -230,10 +232,12 @@ private: std::string mirror_uuid; std::string image_id; librados::IoCtx io_ctx; + MirrorStatusUpdater* mirror_status_updater = nullptr; RemoteImage() { } - RemoteImage(const Peer& peer) : io_ctx(peer.io_ctx) { + RemoteImage(const Peer& peer) + : io_ctx(peer.io_ctx), mirror_status_updater(peer.mirror_status_updater) { } }; diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc index d80e49b8935..9795f8073f1 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.cc +++ b/src/tools/rbd_mirror/InstanceReplayer.cc @@ -103,12 +103,14 @@ void InstanceReplayer::shut_down(Context *on_finish) { } template -void InstanceReplayer::add_peer(std::string peer_uuid, - librados::IoCtx io_ctx) { +void InstanceReplayer::add_peer( + std::string peer_uuid, librados::IoCtx io_ctx, + MirrorStatusUpdater* remote_status_updater) { dout(10) << peer_uuid << dendl; std::lock_guard locker{m_lock}; - auto result = m_peers.insert(Peer(peer_uuid, io_ctx)).second; + auto result = m_peers.insert( + Peer(peer_uuid, io_ctx, remote_status_updater)).second; ceph_assert(result); } @@ -159,7 +161,8 @@ void InstanceReplayer::acquire_image(InstanceWatcher *instance_watcher, // TODO only a single peer is currently supported ceph_assert(m_peers.size() == 1); auto peer = *m_peers.begin(); - image_replayer->add_peer(peer.peer_uuid, peer.io_ctx); + image_replayer->add_peer(peer.peer_uuid, peer.io_ctx, + peer.mirror_status_updater); start_image_replayer(image_replayer); } else { // A duplicate acquire notification implies (1) connection hiccup or diff --git a/src/tools/rbd_mirror/InstanceReplayer.h b/src/tools/rbd_mirror/InstanceReplayer.h index a2648d0f60e..8f32b23daca 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.h +++ b/src/tools/rbd_mirror/InstanceReplayer.h @@ -55,7 +55,8 @@ public: void init(Context *on_finish); void shut_down(Context *on_finish); - void add_peer(std::string peer_uuid, librados::IoCtx io_ctx); + void add_peer(std::string peer_uuid, librados::IoCtx io_ctx, + MirrorStatusUpdater* remote_status_updater); void acquire_image(InstanceWatcher *instance_watcher, const std::string &global_image_id, Context *on_finish); @@ -89,6 +90,8 @@ private: * @endverbatim */ + typedef std::set> Peers; + librados::IoCtx &m_local_io_ctx; std::string m_local_mirror_uuid; Threads *m_threads; diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc index cd6c5780946..b22a547dc76 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.cc +++ b/src/tools/rbd_mirror/NamespaceReplayer.cc @@ -359,7 +359,8 @@ void NamespaceReplayer::handle_init_instance_replayer(int r) { return; } - m_instance_replayer->add_peer(m_remote_mirror_uuid, m_remote_io_ctx); + m_instance_replayer->add_peer(m_remote_mirror_uuid, m_remote_io_ctx, + m_remote_status_updater.get()); init_instance_watcher(); } diff --git a/src/tools/rbd_mirror/Types.h b/src/tools/rbd_mirror/Types.h index ed3b9d8a450..edce547daeb 100644 --- a/src/tools/rbd_mirror/Types.h +++ b/src/tools/rbd_mirror/Types.h @@ -16,6 +16,8 @@ namespace rbd { namespace mirror { +template struct MirrorStatusUpdater; + // Performance counters enum { l_rbd_mirror_first = 27000, @@ -51,16 +53,20 @@ std::ostream &operator<<(std::ostream &, const ImageId &image_id); typedef std::set ImageIds; +template struct Peer { std::string peer_uuid; librados::IoCtx io_ctx; + MirrorStatusUpdater* mirror_status_updater = nullptr; Peer() { } Peer(const std::string &peer_uuid) : peer_uuid(peer_uuid) { } - Peer(const std::string &peer_uuid, librados::IoCtx& io_ctx) - : peer_uuid(peer_uuid), io_ctx(io_ctx) { + Peer(const std::string &peer_uuid, librados::IoCtx& io_ctx, + MirrorStatusUpdater* mirror_status_updater) + : peer_uuid(peer_uuid), io_ctx(io_ctx), + mirror_status_updater(mirror_status_updater) { } inline bool operator<(const Peer &rhs) const { @@ -68,8 +74,6 @@ struct Peer { } }; -typedef std::set Peers; - struct PeerSpec { PeerSpec() = default; PeerSpec(const std::string &uuid, const std::string &cluster_name,