From: Jason Dillaman Date: Wed, 29 Mar 2017 22:04:59 +0000 (-0400) Subject: rbd-mirror: pool watcher should retrieve the mirror uuid X-Git-Tag: v12.0.2~174^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=7cab617a6ae97483c77ba8d72109bc4567605de7;p=ceph-ci.git rbd-mirror: pool watcher should retrieve the mirror uuid Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/test_PoolWatcher.cc b/src/test/rbd_mirror/test_PoolWatcher.cc index 76ba18c565a..dd857423ee0 100644 --- a/src/test/rbd_mirror/test_PoolWatcher.cc +++ b/src/test/rbd_mirror/test_PoolWatcher.cc @@ -75,7 +75,8 @@ public: PoolWatcherListener(TestPoolWatcher *test) : test(test) { } - void handle_update(const ImageIds &added_image_ids, + void handle_update(const std::string &mirror_uuid, + const ImageIds &added_image_ids, const ImageIds &removed_image_ids) override { Mutex::Locker locker(test->m_lock); for (auto &image_id : removed_image_ids) { diff --git a/src/test/rbd_mirror/test_mock_PoolWatcher.cc b/src/test/rbd_mirror/test_mock_PoolWatcher.cc index 334e6c9ae2a..ebb26b9f223 100644 --- a/src/test/rbd_mirror/test_mock_PoolWatcher.cc +++ b/src/test/rbd_mirror/test_mock_PoolWatcher.cc @@ -163,7 +163,8 @@ public: MockListener(TestMockPoolWatcher *test) : test(test) { } - MOCK_METHOD2(handle_update, void(const ImageIds &, const ImageIds &)); + MOCK_METHOD3(handle_update, void(const std::string &, const ImageIds &, + const ImageIds &)); }; TestMockPoolWatcher() : m_lock("TestMockPoolWatcher::m_lock") { @@ -204,9 +205,11 @@ public: } void expect_listener_handle_update(MockListener &mock_listener, + const std::string &mirror_uuid, const ImageIds &added_image_ids, const ImageIds &removed_image_ids) { - EXPECT_CALL(mock_listener, handle_update(added_image_ids, removed_image_ids)) + EXPECT_CALL(mock_listener, handle_update(mirror_uuid, added_image_ids, + removed_image_ids)) .WillOnce(WithoutArgs(Invoke([this]() { Mutex::Locker locker(m_lock); ++m_update_count; @@ -214,6 +217,20 @@ public: }))); } + void expect_mirror_uuid_get(librados::IoCtx &io_ctx, + const std::string &uuid, int r) { + bufferlist out_bl; + ::encode(uuid, out_bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"), + _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([this, out_bl](bufferlist *bl) { + *bl = out_bl; + })), + Return(r))); + } + void expect_dir_list(librados::IoCtx &io_ctx, const std::string &id, const std::string &name, int r) { bufferlist in_bl; @@ -298,9 +315,10 @@ TEST_F(TestMockPoolWatcher, EmptyPool) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, {}, {}); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, mock_listener); @@ -327,9 +345,10 @@ TEST_F(TestMockPoolWatcher, NonEmptyPool) { {"global id 2", "remote id 2", "image name 2"}}; MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, image_ids, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, image_ids, {}); + expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, mock_listener); @@ -370,11 +389,13 @@ TEST_F(TestMockPoolWatcher, NotifyDuringRefresh) { expect_dir_list(m_remote_io_ctx, "remote id 3", "image name 3", 0); expect_dir_list(m_remote_io_ctx, "dummy", "", -ENOENT); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + MockListener mock_listener(this); image_ids = { {"global id 1", "remote id 1a", "image name 1a"}, {"global id 3", "remote id 3", "image name 3"}}; - expect_listener_handle_update(mock_listener, image_ids, {}); + expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, mock_listener); @@ -417,13 +438,15 @@ TEST_F(TestMockPoolWatcher, Notify) { {"global id 2", "remote id 2", "image name 2"}}; MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, image_ids, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) .WillOnce(Invoke([this](Context *ctx, int r) { m_threads->work_queue->queue(ctx, r); })); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, image_ids, {}); + expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); Context *notify_ctx = nullptr; EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) @@ -437,7 +460,7 @@ TEST_F(TestMockPoolWatcher, Notify) { expect_dir_list(m_remote_io_ctx, "remote id 3", "image name 3", 0); expect_dir_list(m_remote_io_ctx, "dummy", "", -ENOENT); expect_listener_handle_update( - mock_listener, + mock_listener, "remote uuid", {{"global id 1", "remote id 1a", "image name 1a"}, {"global id 3", "remote id 3", "image name 3"}}, {{"global id 1", "remote id 1", "image name 1"}, @@ -509,9 +532,10 @@ TEST_F(TestMockPoolWatcher, RegisterWatcherMissing) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, {}, {}); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, mock_listener); @@ -539,9 +563,10 @@ TEST_F(TestMockPoolWatcher, RegisterWatcherError) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, {}, {}); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, mock_listener); @@ -589,9 +614,10 @@ TEST_F(TestMockPoolWatcher, RefreshMissing) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, -ENOENT); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, {}, {}); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, mock_listener); @@ -619,9 +645,99 @@ TEST_F(TestMockPoolWatcher, RefreshError) { expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, GetMirrorUuidBlacklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", -EBLACKLISTED); + + MockListener mock_listener(this); + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(-EBLACKLISTED, ctx.wait()); + ASSERT_TRUE(mock_pool_watcher.is_blacklisted()); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, GetMirrorUuidMissing) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "", -ENOENT); + expect_timer_add_event(mock_threads); + + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, {}, {}); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(-ENOENT, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, GetMirrorUuidError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", -EINVAL); + expect_timer_add_event(mock_threads); + + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, mock_listener); @@ -645,14 +761,17 @@ TEST_F(TestMockPoolWatcher, Rewatch) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, {}, {}); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); expect_timer_add_event(mock_threads); expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); expect_refresh_images(mock_refresh_images_request, {{"global id", "image id", "name"}}, 0); - expect_listener_handle_update(mock_listener, {{"global id", "image id", "name"}}, {}); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + expect_listener_handle_update(mock_listener, "remote uuid", + {{"global id", "image id", "name"}}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, mock_listener); @@ -679,9 +798,10 @@ TEST_F(TestMockPoolWatcher, RewatchBlacklist) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, {}, {}); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, mock_listener); @@ -708,14 +828,17 @@ TEST_F(TestMockPoolWatcher, RewatchError) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, {}, {}); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); expect_timer_add_event(mock_threads); expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); expect_refresh_images(mock_refresh_images_request, {{"global id", "image id", "name"}}, 0); - expect_listener_handle_update(mock_listener, {{"global id", "image id", "name"}}, {}); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + expect_listener_handle_update(mock_listener, "remote uuid", + {{"global id", "image id", "name"}}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, mock_listener); @@ -742,9 +865,10 @@ TEST_F(TestMockPoolWatcher, GetImageNameBlacklist) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, {}, {}); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); expect_dir_list(m_remote_io_ctx, "remote id", "image name", -EBLACKLISTED); @@ -780,16 +904,19 @@ TEST_F(TestMockPoolWatcher, GetImageNameError) { MockRefreshImagesRequest mock_refresh_images_request; expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, {}, {}); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); expect_dir_list(m_remote_io_ctx, "remote id", "image name", -EINVAL); expect_timer_add_event(mock_threads); expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); expect_refresh_images(mock_refresh_images_request, {{"global id", "remote id", "name"}}, 0); - expect_listener_handle_update(mock_listener, {{"global id", "remote id", "name"}}, {}); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + expect_listener_handle_update(mock_listener, "remote uuid", + {{"global id", "remote id", "name"}}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, mock_listener); @@ -824,23 +951,69 @@ TEST_F(TestMockPoolWatcher, DeferredRefresh) { MirroringWatcher::get_instance().handle_rewatch_complete(0); mock_refresh_images_request.on_finish->complete(0); })); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); expect_timer_add_event(mock_threads); + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); MockListener mock_listener(this); - expect_listener_handle_update(mock_listener, {}, {}); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, mock_listener); C_SaferCond ctx; mock_pool_watcher.init(&ctx); ASSERT_EQ(0, ctx.wait()); - ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); } +TEST_F(TestMockPoolWatcher, MirrorUuidUpdated) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + ImageIds image_ids{ + {"global id 1", "remote id 1", "image name 1"}, + {"global id 2", "remote id 2", "image name 2"}}; + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, image_ids, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + + expect_timer_add_event(mock_threads); + ImageIds new_image_ids{ + {"global id 1", "remote id 1", "image name 1"}}; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, new_image_ids, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "updated uuid", 0); + expect_listener_handle_update(mock_listener, "remote uuid", {}, image_ids); + expect_listener_handle_update(mock_listener, "updated uuid", new_image_ids, + {}); + + MirroringWatcher::get_instance().handle_rewatch_complete(0); + ASSERT_TRUE(wait_for_update(2)); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} } // namespace mirror } // namespace rbd diff --git a/src/tools/rbd_mirror/PoolWatcher.cc b/src/tools/rbd_mirror/PoolWatcher.cc index aa2016f4cbe..7bdd8cdb09e 100644 --- a/src/tools/rbd_mirror/PoolWatcher.cc +++ b/src/tools/rbd_mirror/PoolWatcher.cc @@ -51,7 +51,8 @@ public: } void handle_mode_updated(cls::rbd::MirrorMode mirror_mode) override { - // do nothing + // invalidate all image state and refresh the pool contents + m_pool_watcher->schedule_refresh_images(5); } void handle_image_updated(cls::rbd::MirrorImageState state, @@ -244,6 +245,66 @@ template void PoolWatcher::handle_refresh_images(int r) { dout(5) << "r=" << r << dendl; + bool retry_refresh = false; + Context *on_init_finish = nullptr; + { + Mutex::Locker locker(m_lock); + assert(m_image_ids_invalid); + assert(m_refresh_in_progress); + + if (r >= 0) { + m_pending_image_ids = std::move(m_refresh_image_ids); + } else if (r == -EBLACKLISTED) { + dout(0) << "detected client is blacklisted during image refresh" << dendl; + + m_blacklisted = true; + m_refresh_in_progress = false; + std::swap(on_init_finish, m_on_init_finish); + } else if (r == -ENOENT) { + dout(5) << "mirroring directory not found" << dendl; + m_pending_image_ids.clear(); + r = 0; + } else { + m_refresh_in_progress = false; + retry_refresh = true; + } + } + + if (retry_refresh) { + derr << "failed to retrieve mirroring directory: " << cpp_strerror(r) + << dendl; + schedule_refresh_images(10); + } else if (r >= 0) { + get_mirror_uuid(); + return; + } + + m_async_op_tracker.finish_op(); + if (on_init_finish != nullptr) { + assert(r == -EBLACKLISTED); + on_init_finish->complete(r); + } +} + +template +void PoolWatcher::get_mirror_uuid() { + dout(5) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_uuid_get_start(&op); + + m_out_bl.clear(); + librados::AioCompletion *aio_comp = create_rados_callback< + PoolWatcher, &PoolWatcher::handle_get_mirror_uuid>(this); + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + assert(r == 0); + aio_comp->release(); +} + +template +void PoolWatcher::handle_get_mirror_uuid(int r) { + dout(5) << "r=" << r << dendl; + bool deferred_refresh = false; bool retry_refresh = false; Context *on_init_finish = nullptr; @@ -253,12 +314,22 @@ void PoolWatcher::handle_refresh_images(int r) { assert(m_refresh_in_progress); m_refresh_in_progress = false; + m_pending_mirror_uuid = ""; + if (r >= 0) { + bufferlist::iterator it = m_out_bl.begin(); + r = librbd::cls_client::mirror_uuid_get_finish( + &it, &m_pending_mirror_uuid); + } + if (r >= 0 && m_pending_mirror_uuid.empty()) { + r = -ENOENT; + } + if (m_deferred_refresh) { // need to refresh -- skip the notification deferred_refresh = true; } else if (r >= 0) { + dout(10) << "mirror_uuid=" << m_pending_mirror_uuid << dendl; m_image_ids_invalid = false; - m_pending_image_ids = m_refresh_image_ids; std::swap(on_init_finish, m_on_init_finish); schedule_listener(); } else if (r == -EBLACKLISTED) { @@ -267,12 +338,9 @@ void PoolWatcher::handle_refresh_images(int r) { m_blacklisted = true; std::swap(on_init_finish, m_on_init_finish); } else if (r == -ENOENT) { - dout(5) << "mirroring directory not found" << dendl; - m_image_ids_invalid = false; - m_pending_image_ids.clear(); + dout(5) << "mirroring uuid not found" << dendl; std::swap(on_init_finish, m_on_init_finish); - r = 0; - schedule_listener(); + retry_refresh = true; } else { retry_refresh = true; } @@ -282,7 +350,7 @@ void PoolWatcher::handle_refresh_images(int r) { dout(5) << "scheduling deferred refresh" << dendl; schedule_refresh_images(0); } else if (retry_refresh) { - derr << "failed to retrieve mirroring directory: " << cpp_strerror(r) + derr << "failed to retrieve mirror uuid: " << cpp_strerror(r) << dendl; schedule_refresh_images(10); } @@ -495,8 +563,33 @@ template void PoolWatcher::notify_listener() { dout(10) << dendl; + std::string mirror_uuid; ImageIds added_image_ids; ImageIds removed_image_ids; + { + Mutex::Locker locker(m_lock); + assert(m_notify_listener_in_progress); + + // if the mirror uuid is updated, treat it as the removal of all + // images in the pool + if (m_mirror_uuid != m_pending_mirror_uuid) { + if (!m_mirror_uuid.empty()) { + dout(0) << "mirror uuid updated:" + << "old=" << m_mirror_uuid << ", " + << "new=" << m_pending_mirror_uuid << dendl; + } + + mirror_uuid = m_mirror_uuid; + removed_image_ids = std::move(m_image_ids); + m_image_ids.clear(); + } + } + + if (!removed_image_ids.empty()) { + m_listener.handle_update(mirror_uuid, {}, removed_image_ids); + removed_image_ids.clear(); + } + { Mutex::Locker locker(m_lock); assert(m_notify_listener_in_progress); @@ -553,9 +646,12 @@ void PoolWatcher::notify_listener() { m_pending_updates = false; m_image_ids = m_pending_image_ids; + + m_mirror_uuid = m_pending_mirror_uuid; + mirror_uuid = m_mirror_uuid; } - m_listener.handle_update(added_image_ids, removed_image_ids); + m_listener.handle_update(mirror_uuid, added_image_ids, removed_image_ids); { Mutex::Locker locker(m_lock); diff --git a/src/tools/rbd_mirror/PoolWatcher.h b/src/tools/rbd_mirror/PoolWatcher.h index a8aa32948bb..747a0ee5dc8 100644 --- a/src/tools/rbd_mirror/PoolWatcher.h +++ b/src/tools/rbd_mirror/PoolWatcher.h @@ -38,7 +38,8 @@ public: virtual ~Listener() { } - virtual void handle_update(const ImageIds &added_image_ids, + virtual void handle_update(const std::string &mirror_uuid, + const ImageIds &added_image_ids, const ImageIds &removed_image_ids) = 0; }; @@ -73,6 +74,9 @@ private: * |/----------------------------\ | * | | | * v | | + * GET_MIRROR_UUID | | + * | | | + * v | | * NOTIFY_LISTENER | | * | | | * v | | @@ -134,6 +138,7 @@ private: Context *m_on_init_finish = nullptr; ImageIds m_image_ids; + std::string m_mirror_uuid; bool m_pending_updates = false; bool m_notify_listener_in_progress = false; @@ -141,6 +146,8 @@ private: ImageIds m_pending_added_image_ids; ImageIds m_pending_removed_image_ids; + std::string m_pending_mirror_uuid; + MirroringWatcher *m_mirroring_watcher; Context *m_timer_ctx = nullptr; @@ -166,6 +173,9 @@ private: void schedule_refresh_images(double interval); void process_refresh_images(); + void get_mirror_uuid(); + void handle_get_mirror_uuid(int r); + void handle_rewatch_complete(int r); void handle_image_updated(const std::string &remote_image_id, const std::string &global_image_id, diff --git a/src/tools/rbd_mirror/Replayer.cc b/src/tools/rbd_mirror/Replayer.cc index f8ee473e192..953db35f63e 100644 --- a/src/tools/rbd_mirror/Replayer.cc +++ b/src/tools/rbd_mirror/Replayer.cc @@ -568,8 +568,10 @@ void Replayer::release_leader() m_leader_watcher->release_leader(); } -void Replayer::handle_update(const ImageIds &added_image_ids, +void Replayer::handle_update(const std::string &mirror_uuid, + const ImageIds &added_image_ids, const ImageIds &removed_image_ids) { + assert(!mirror_uuid.empty()); if (m_stopping.read()) { return; } @@ -605,9 +607,7 @@ void Replayer::handle_update(const ImageIds &added_image_ids, for (auto &image_id : removed_image_ids) { auto image_it = m_image_replayers.find(image_id.global_id); if (image_it != m_image_replayers.end()) { - assert(!m_remote_mirror_uuid.empty()); - image_it->second->remove_remote_image(m_remote_mirror_uuid, - image_id.id); + image_it->second->remove_remote_image(mirror_uuid, image_id.id); if (image_it->second->is_running()) { dout(20) << "stop image replayer for remote image " @@ -647,16 +647,6 @@ void Replayer::handle_update(const ImageIds &added_image_ids, return; } - std::string remote_mirror_uuid; - r = librbd::cls_client::mirror_uuid_get(&m_remote_io_ctx, - &remote_mirror_uuid); - if (r < 0 || remote_mirror_uuid.empty()) { - derr << "failed to retrieve remote mirror uuid from pool " - << m_remote_io_ctx.get_pool_name() << ": " << cpp_strerror(r) << dendl; - return; - } - m_remote_mirror_uuid = remote_mirror_uuid; - // start replayers for newly added remote image sources for (auto &image_id : added_image_ids) { auto it = m_image_replayers.find(image_id.global_id); @@ -672,7 +662,7 @@ void Replayer::handle_update(const ImageIds &added_image_ids, std::make_pair(image_id.global_id, std::move(image_replayer))).first; } - it->second->add_remote_image(remote_mirror_uuid, image_id.id, + it->second->add_remote_image(mirror_uuid, image_id.id, m_remote_io_ctx); if (!it->second->is_running()) { dout(20) << "starting image replayer for remote image " diff --git a/src/tools/rbd_mirror/Replayer.h b/src/tools/rbd_mirror/Replayer.h index f503d1495fe..6eb3753aa3b 100644 --- a/src/tools/rbd_mirror/Replayer.h +++ b/src/tools/rbd_mirror/Replayer.h @@ -66,15 +66,17 @@ private: PoolWatcherListener(Replayer *replayer) : replayer(replayer) { } - void handle_update(const ImageIds &added_image_ids, + void handle_update(const std::string &mirror_uuid, + const ImageIds &added_image_ids, const ImageIds &removed_image_ids) override { - replayer->handle_update(added_image_ids, removed_image_ids); + replayer->handle_update(mirror_uuid, added_image_ids, removed_image_ids); } }; struct C_RefreshLocalImages; - void handle_update(const ImageIds &added_image_ids, + void handle_update(const std::string &mirror_uuid, + const ImageIds &added_image_ids, const ImageIds &removed_image_ids); void start_image_replayer(unique_ptr > &image_replayer); @@ -117,8 +119,6 @@ private: int64_t m_local_pool_id = -1; int64_t m_remote_pool_id = -1; - std::string m_remote_mirror_uuid; - PoolWatcherListener m_pool_watcher_listener; std::unique_ptr > m_pool_watcher;