From dedf4c63b76917029698f97a949dad5ee8787130 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 15 Jan 2020 22:23:21 -0500 Subject: [PATCH] rbd-mirror: pool watcher no longer needs to query mirror uuid This is periodically polled now via the RemotePoolPoller and it detects changes to the mirror uuid. Fixes: https://tracker.ceph.com/issues/43537 Signed-off-by: Jason Dillaman --- src/test/rbd_mirror/test_PoolWatcher.cc | 2 +- .../rbd_mirror/test_mock_NamespaceReplayer.cc | 1 + src/test/rbd_mirror/test_mock_PoolWatcher.cc | 190 ++---------------- src/tools/rbd_mirror/NamespaceReplayer.cc | 6 +- src/tools/rbd_mirror/PoolWatcher.cc | 130 +++--------- src/tools/rbd_mirror/PoolWatcher.h | 23 +-- 6 files changed, 55 insertions(+), 297 deletions(-) diff --git a/src/test/rbd_mirror/test_PoolWatcher.cc b/src/test/rbd_mirror/test_PoolWatcher.cc index 78cab0516adf9..493c675d0c4cb 100644 --- a/src/test/rbd_mirror/test_PoolWatcher.cc +++ b/src/test/rbd_mirror/test_PoolWatcher.cc @@ -100,7 +100,7 @@ public: ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx)); ioctx.application_enable("rbd", true); - m_pool_watcher.reset(new PoolWatcher<>(m_threads, ioctx, + m_pool_watcher.reset(new PoolWatcher<>(m_threads, ioctx, "mirror uuid", m_pool_watcher_listener)); if (enable_mirroring) { diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc index 99f27237ad92e..53729598d7be3 100644 --- a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc @@ -197,6 +197,7 @@ struct PoolWatcher { static PoolWatcher *create(Threads *threads, librados::IoCtx &ioctx, + const std::string& mirror_uuid, pool_watcher::Listener& listener) { auto pool_id = ioctx.get_id(); ceph_assert(s_instances.count(pool_id)); diff --git a/src/test/rbd_mirror/test_mock_PoolWatcher.cc b/src/test/rbd_mirror/test_mock_PoolWatcher.cc index faacf2c5b1a43..0273133da81b8 100644 --- a/src/test/rbd_mirror/test_mock_PoolWatcher.cc +++ b/src/test/rbd_mirror/test_mock_PoolWatcher.cc @@ -222,20 +222,6 @@ public: }))); } - void expect_mirror_uuid_get(librados::IoCtx &io_ctx, - const std::string &uuid, int r) { - bufferlist out_bl; - encode(uuid, out_bl); - - EXPECT_CALL(get_mock_io_ctx(io_ctx), - exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"), - _, _, _)) - .WillOnce(DoAll(WithArg<5>(Invoke([out_bl](bufferlist *bl) { - *bl = out_bl; - })), - Return(r))); - } - void expect_timer_add_event(MockThreads &mock_threads) { EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) { @@ -282,13 +268,12 @@ TEST_F(TestMockPoolWatcher, EmptyPool) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(0, ctx.wait()); @@ -312,13 +297,12 @@ TEST_F(TestMockPoolWatcher, NonEmptyPool) { {"global id 2", "remote id 2"}}; MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, image_ids, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(0, ctx.wait()); @@ -352,7 +336,6 @@ TEST_F(TestMockPoolWatcher, NotifyDuringRefresh) { m_cond.notify_all(); })); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); image_ids = { @@ -361,7 +344,7 @@ TEST_F(TestMockPoolWatcher, NotifyDuringRefresh) { expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); mock_pool_watcher.init(nullptr); { @@ -396,7 +379,6 @@ TEST_F(TestMockPoolWatcher, Notify) { {"global id 2", "remote id 2"}}; MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, image_ids, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) .WillOnce(Invoke([this](Context *ctx, int r) { @@ -420,7 +402,7 @@ TEST_F(TestMockPoolWatcher, Notify) { {{"global id 1", "remote id 1"}, {"global id 2", "remote id 2"}}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(0, ctx.wait()); @@ -457,7 +439,7 @@ TEST_F(TestMockPoolWatcher, RegisterWatcherBlacklist) { MockListener mock_listener(this); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(-EBLACKLISTED, ctx.wait()); @@ -482,13 +464,12 @@ TEST_F(TestMockPoolWatcher, RegisterWatcherMissing) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(-ENOENT, ctx.wait()); @@ -513,13 +494,12 @@ TEST_F(TestMockPoolWatcher, RegisterWatcherError) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(0, ctx.wait()); @@ -543,7 +523,7 @@ TEST_F(TestMockPoolWatcher, RefreshBlacklist) { MockListener mock_listener(this); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(-EBLACKLISTED, ctx.wait()); @@ -564,13 +544,12 @@ TEST_F(TestMockPoolWatcher, RefreshMissing) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, -ENOENT); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(0, ctx.wait()); @@ -595,102 +574,12 @@ TEST_F(TestMockPoolWatcher, RefreshError) { expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); - C_SaferCond ctx; - mock_pool_watcher.init(&ctx); - ASSERT_EQ(0, ctx.wait()); - - ASSERT_TRUE(wait_for_update(1)); - expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); - ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); -} - -TEST_F(TestMockPoolWatcher, GetMirrorUuidBlacklist) { - MockThreads mock_threads(m_threads); - expect_work_queue(mock_threads); - - InSequence seq; - MockMirroringWatcher mock_mirroring_watcher; - expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); - expect_mirroring_watcher_register(mock_mirroring_watcher, 0); - - MockRefreshImagesRequest mock_refresh_images_request; - expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", -EBLACKLISTED); - - MockListener mock_listener(this); - MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); - C_SaferCond ctx; - mock_pool_watcher.init(&ctx); - ASSERT_EQ(-EBLACKLISTED, ctx.wait()); - ASSERT_TRUE(mock_pool_watcher.is_blacklisted()); - - expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); - ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); -} - -TEST_F(TestMockPoolWatcher, GetMirrorUuidMissing) { - MockThreads mock_threads(m_threads); - expect_work_queue(mock_threads); - - InSequence seq; - MockMirroringWatcher mock_mirroring_watcher; - expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); - expect_mirroring_watcher_register(mock_mirroring_watcher, 0); - - MockRefreshImagesRequest mock_refresh_images_request; - expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "", -ENOENT); - expect_timer_add_event(mock_threads); - - expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); - expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); - - MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); - - MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); - C_SaferCond ctx; - mock_pool_watcher.init(&ctx); - ASSERT_EQ(-ENOENT, ctx.wait()); - - ASSERT_TRUE(wait_for_update(1)); - expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); - ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); -} - -TEST_F(TestMockPoolWatcher, GetMirrorUuidError) { - MockThreads mock_threads(m_threads); - expect_work_queue(mock_threads); - - InSequence seq; - MockMirroringWatcher mock_mirroring_watcher; - expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); - expect_mirroring_watcher_register(mock_mirroring_watcher, 0); - - MockRefreshImagesRequest mock_refresh_images_request; - expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", -EINVAL); - expect_timer_add_event(mock_threads); - - expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); - expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); - - MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); - - MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(0, ctx.wait()); @@ -711,7 +600,6 @@ TEST_F(TestMockPoolWatcher, Rewatch) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); @@ -719,12 +607,11 @@ TEST_F(TestMockPoolWatcher, Rewatch) { expect_timer_add_event(mock_threads); expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); expect_refresh_images(mock_refresh_images_request, {{"global id", "image id"}}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); expect_listener_handle_update(mock_listener, "remote uuid", {{"global id", "image id"}}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(0, ctx.wait()); @@ -748,13 +635,12 @@ TEST_F(TestMockPoolWatcher, RewatchBlacklist) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(0, ctx.wait()); @@ -778,7 +664,6 @@ TEST_F(TestMockPoolWatcher, RewatchError) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); @@ -786,12 +671,11 @@ TEST_F(TestMockPoolWatcher, RewatchError) { expect_timer_add_event(mock_threads); expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); expect_refresh_images(mock_refresh_images_request, {{"global id", "image id"}}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); expect_listener_handle_update(mock_listener, "remote uuid", {{"global id", "image id"}}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(0, ctx.wait()); @@ -821,67 +705,21 @@ TEST_F(TestMockPoolWatcher, DeferredRefresh) { MirroringWatcher::get_instance().handle_rewatch_complete(0); mock_refresh_images_request.on_finish->complete(0); })); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); expect_timer_add_event(mock_threads); expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); expect_refresh_images(mock_refresh_images_request, {}, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); - C_SaferCond ctx; - mock_pool_watcher.init(&ctx); - ASSERT_EQ(0, ctx.wait()); - ASSERT_TRUE(wait_for_update(1)); - - expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); - ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); -} - -TEST_F(TestMockPoolWatcher, MirrorUuidUpdated) { - MockThreads mock_threads(m_threads); - expect_work_queue(mock_threads); - - InSequence seq; - MockMirroringWatcher mock_mirroring_watcher; - expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); - expect_mirroring_watcher_register(mock_mirroring_watcher, 0); - - ImageIds image_ids{ - {"global id 1", "remote id 1"}, - {"global id 2", "remote id 2"}}; - MockRefreshImagesRequest mock_refresh_images_request; - expect_refresh_images(mock_refresh_images_request, image_ids, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); - - MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); - - MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, - mock_listener); + "remote uuid", mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(0, ctx.wait()); - ASSERT_TRUE(wait_for_update(1)); - expect_timer_add_event(mock_threads); - ImageIds new_image_ids{ - {"global id 1", "remote id 1"}}; - expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); - expect_refresh_images(mock_refresh_images_request, new_image_ids, 0); - expect_mirror_uuid_get(m_remote_io_ctx, "updated uuid", 0); - expect_listener_handle_update(mock_listener, "remote uuid", {}, image_ids); - expect_listener_handle_update(mock_listener, "updated uuid", new_image_ids, - {}); - - MirroringWatcher::get_instance().handle_rewatch_complete(0); - ASSERT_TRUE(wait_for_update(2)); - expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); } diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc index 77bc7c626ee00..3d86ff1fe9294 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.cc +++ b/src/tools/rbd_mirror/NamespaceReplayer.cc @@ -608,7 +608,8 @@ void NamespaceReplayer::init_local_pool_watcher(Context *on_finish) { std::lock_guard locker{m_lock}; ceph_assert(!m_local_pool_watcher); m_local_pool_watcher.reset(PoolWatcher::create( - m_threads, m_local_io_ctx, m_local_pool_watcher_listener)); + m_threads, m_local_io_ctx, m_local_mirror_uuid, + m_local_pool_watcher_listener)); // ensure the initial set of local images is up-to-date // after acquiring the leader role @@ -642,7 +643,8 @@ void NamespaceReplayer::init_remote_pool_watcher(Context *on_finish) { std::lock_guard locker{m_lock}; ceph_assert(!m_remote_pool_watcher); m_remote_pool_watcher.reset(PoolWatcher::create( - m_threads, m_remote_io_ctx, m_remote_pool_watcher_listener)); + m_threads, m_remote_io_ctx, m_remote_pool_meta.mirror_uuid, + m_remote_pool_watcher_listener)); auto ctx = new LambdaContext([this, on_finish](int r) { handle_init_remote_pool_watcher(r, on_finish); diff --git a/src/tools/rbd_mirror/PoolWatcher.cc b/src/tools/rbd_mirror/PoolWatcher.cc index 4bc516f5ebacb..359ca95d53772 100644 --- a/src/tools/rbd_mirror/PoolWatcher.cc +++ b/src/tools/rbd_mirror/PoolWatcher.cc @@ -56,10 +56,10 @@ public: } void handle_image_updated(cls::rbd::MirrorImageState state, - const std::string &remote_image_id, + const std::string &image_id, const std::string &global_image_id) override { bool enabled = (state == cls::rbd::MIRROR_IMAGE_STATE_ENABLED); - m_pool_watcher->handle_image_updated(remote_image_id, global_image_id, + m_pool_watcher->handle_image_updated(image_id, global_image_id, enabled); } @@ -68,11 +68,17 @@ private: }; template -PoolWatcher::PoolWatcher(Threads *threads, librados::IoCtx &remote_io_ctx, +PoolWatcher::PoolWatcher(Threads *threads, + librados::IoCtx &io_ctx, + const std::string& mirror_uuid, pool_watcher::Listener &listener) - : m_threads(threads), m_remote_io_ctx(remote_io_ctx), m_listener(listener), - m_lock(ceph::make_mutex(librbd::util::unique_lock_name("rbd::mirror::PoolWatcher", this))) { - m_mirroring_watcher = new MirroringWatcher(m_remote_io_ctx, + : m_threads(threads), + m_io_ctx(io_ctx), + m_mirror_uuid(mirror_uuid), + m_listener(listener), + m_lock(ceph::make_mutex(librbd::util::unique_lock_name( + "rbd::mirror::PoolWatcher", this))) { + m_mirroring_watcher = new MirroringWatcher(m_io_ctx, m_threads->work_queue, this); } @@ -225,7 +231,7 @@ void PoolWatcher::refresh_images() { m_refresh_image_ids.clear(); Context *ctx = create_context_callback< PoolWatcher, &PoolWatcher::handle_refresh_images>(this); - auto req = pool_watcher::RefreshImagesRequest::create(m_remote_io_ctx, + auto req = pool_watcher::RefreshImagesRequest::create(m_io_ctx, &m_refresh_image_ids, ctx); req->send(); @@ -235,66 +241,6 @@ template void PoolWatcher::handle_refresh_images(int r) { dout(5) << "r=" << r << dendl; - bool retry_refresh = false; - Context *on_init_finish = nullptr; - { - std::lock_guard locker{m_lock}; - ceph_assert(m_image_ids_invalid); - ceph_assert(m_refresh_in_progress); - - if (r >= 0) { - m_pending_image_ids = std::move(m_refresh_image_ids); - } else if (r == -EBLACKLISTED) { - dout(0) << "detected client is blacklisted during image refresh" << dendl; - - m_blacklisted = true; - m_refresh_in_progress = false; - std::swap(on_init_finish, m_on_init_finish); - } else if (r == -ENOENT) { - dout(5) << "mirroring directory not found" << dendl; - m_pending_image_ids.clear(); - r = 0; - } else { - m_refresh_in_progress = false; - retry_refresh = true; - } - } - - if (retry_refresh) { - derr << "failed to retrieve mirroring directory: " << cpp_strerror(r) - << dendl; - schedule_refresh_images(10); - } else if (r >= 0) { - get_mirror_uuid(); - return; - } - - m_async_op_tracker.finish_op(); - if (on_init_finish != nullptr) { - ceph_assert(r == -EBLACKLISTED); - on_init_finish->complete(r); - } -} - -template -void PoolWatcher::get_mirror_uuid() { - dout(5) << dendl; - - librados::ObjectReadOperation op; - librbd::cls_client::mirror_uuid_get_start(&op); - - m_out_bl.clear(); - librados::AioCompletion *aio_comp = create_rados_callback< - PoolWatcher, &PoolWatcher::handle_get_mirror_uuid>(this); - int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); - ceph_assert(r == 0); - aio_comp->release(); -} - -template -void PoolWatcher::handle_get_mirror_uuid(int r) { - dout(5) << "r=" << r << dendl; - bool deferred_refresh = false; bool retry_refresh = false; Context *on_init_finish = nullptr; @@ -304,33 +250,26 @@ void PoolWatcher::handle_get_mirror_uuid(int r) { ceph_assert(m_refresh_in_progress); m_refresh_in_progress = false; - m_pending_mirror_uuid = ""; - if (r >= 0) { - auto it = m_out_bl.cbegin(); - r = librbd::cls_client::mirror_uuid_get_finish( - &it, &m_pending_mirror_uuid); - } - if (r >= 0 && m_pending_mirror_uuid.empty()) { - r = -ENOENT; + if (r == -ENOENT) { + dout(5) << "mirroring directory not found" << dendl; + r = 0; + m_refresh_image_ids.clear(); } if (m_deferred_refresh) { // need to refresh -- skip the notification deferred_refresh = true; } else if (r >= 0) { - dout(10) << "mirror_uuid=" << m_pending_mirror_uuid << dendl; + m_pending_image_ids = std::move(m_refresh_image_ids); m_image_ids_invalid = false; std::swap(on_init_finish, m_on_init_finish); + schedule_listener(); } else if (r == -EBLACKLISTED) { dout(0) << "detected client is blacklisted during image refresh" << dendl; m_blacklisted = true; std::swap(on_init_finish, m_on_init_finish); - } else if (r == -ENOENT) { - dout(5) << "mirroring uuid not found" << dendl; - std::swap(on_init_finish, m_on_init_finish); - retry_refresh = true; } else { retry_refresh = true; } @@ -340,7 +279,7 @@ void PoolWatcher::handle_get_mirror_uuid(int r) { dout(5) << "scheduling deferred refresh" << dendl; schedule_refresh_images(0); } else if (retry_refresh) { - derr << "failed to retrieve mirror uuid: " << cpp_strerror(r) + derr << "failed to retrieve mirroring directory: " << cpp_strerror(r) << dendl; schedule_refresh_images(10); } @@ -391,15 +330,15 @@ void PoolWatcher::handle_rewatch_complete(int r) { } template -void PoolWatcher::handle_image_updated(const std::string &remote_image_id, - const std::string &global_image_id, - bool enabled) { - dout(10) << "remote_image_id=" << remote_image_id << ", " +void PoolWatcher::handle_image_updated(const std::string &id, + const std::string &global_image_id, + bool enabled) { + dout(10) << "image_id=" << id << ", " << "global_image_id=" << global_image_id << ", " << "enabled=" << enabled << dendl; std::lock_guard locker{m_lock}; - ImageId image_id(global_image_id, remote_image_id); + ImageId image_id(global_image_id, id); m_pending_added_image_ids.erase(image_id); m_pending_removed_image_ids.erase(image_id); @@ -464,20 +403,6 @@ void PoolWatcher::notify_listener() { { std::lock_guard locker{m_lock}; ceph_assert(m_notify_listener_in_progress); - - // if the mirror uuid is updated, treat it as the removal of all - // images in the pool - if (m_mirror_uuid != m_pending_mirror_uuid) { - if (!m_mirror_uuid.empty()) { - dout(0) << "mirror uuid updated:" - << "old=" << m_mirror_uuid << ", " - << "new=" << m_pending_mirror_uuid << dendl; - } - - mirror_uuid = m_mirror_uuid; - removed_image_ids = std::move(m_image_ids); - m_image_ids.clear(); - } } if (!removed_image_ids.empty()) { @@ -526,12 +451,9 @@ void PoolWatcher::notify_listener() { m_pending_updates = false; m_image_ids = m_pending_image_ids; - - m_mirror_uuid = m_pending_mirror_uuid; - mirror_uuid = m_mirror_uuid; } - m_listener.handle_update(mirror_uuid, std::move(added_image_ids), + m_listener.handle_update(m_mirror_uuid, std::move(added_image_ids), std::move(removed_image_ids)); { diff --git a/src/tools/rbd_mirror/PoolWatcher.h b/src/tools/rbd_mirror/PoolWatcher.h index ec21e343ece3b..c5c84731da8cc 100644 --- a/src/tools/rbd_mirror/PoolWatcher.h +++ b/src/tools/rbd_mirror/PoolWatcher.h @@ -34,12 +34,15 @@ template class PoolWatcher { public: static PoolWatcher* create(Threads *threads, - librados::IoCtx &remote_io_ctx, + librados::IoCtx &io_ctx, + const std::string& mirror_uuid, pool_watcher::Listener &listener) { - return new PoolWatcher(threads, remote_io_ctx, listener); + return new PoolWatcher(threads, io_ctx, mirror_uuid, listener); } - PoolWatcher(Threads *threads, librados::IoCtx &remote_io_ctx, + PoolWatcher(Threads *threads, + librados::IoCtx &io_ctx, + const std::string& mirror_uuid, pool_watcher::Listener &listener); ~PoolWatcher(); PoolWatcher(const PoolWatcher&) = delete; @@ -75,9 +78,6 @@ private: * |/----------------------------\ | * | | | * v | | - * GET_MIRROR_UUID | | - * | | | - * v | | * NOTIFY_LISTENER | | * | | | * v | | @@ -103,7 +103,8 @@ private: class MirroringWatcher; Threads *m_threads; - librados::IoCtx m_remote_io_ctx; + librados::IoCtx m_io_ctx; + std::string m_mirror_uuid; pool_watcher::Listener &m_listener; ImageIds m_refresh_image_ids; @@ -114,7 +115,6 @@ private: Context *m_on_init_finish = nullptr; ImageIds m_image_ids; - std::string m_mirror_uuid; bool m_pending_updates = false; bool m_notify_listener_in_progress = false; @@ -122,8 +122,6 @@ private: ImageIds m_pending_added_image_ids; ImageIds m_pending_removed_image_ids; - std::string m_pending_mirror_uuid; - MirroringWatcher *m_mirroring_watcher; Context *m_timer_ctx = nullptr; @@ -145,11 +143,8 @@ private: void schedule_refresh_images(double interval); void process_refresh_images(); - void get_mirror_uuid(); - void handle_get_mirror_uuid(int r); - void handle_rewatch_complete(int r); - void handle_image_updated(const std::string &remote_image_id, + void handle_image_updated(const std::string &image_id, const std::string &global_image_id, bool enabled); -- 2.39.5