From 4b7c194b007a5f3ea6500d37f813931b58ff3c85 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Fri, 9 Mar 2018 10:25:32 -0500 Subject: [PATCH] rbd-mirror: image map listener should provide a callback context The context wraps the necessary data and provides a pass-through chain to the instance notification helpers. Signed-off-by: Jason Dillaman --- src/test/rbd_mirror/test_mock_ImageMap.cc | 366 +++++++++++++++------- src/tools/rbd_mirror/ImageMap.cc | 38 ++- src/tools/rbd_mirror/ImageMap.h | 7 +- src/tools/rbd_mirror/image_map/Types.h | 11 +- 4 files changed, 292 insertions(+), 130 deletions(-) diff --git a/src/test/rbd_mirror/test_mock_ImageMap.cc b/src/test/rbd_mirror/test_mock_ImageMap.cc index 090fb94f74c22..f9aa231ed247f 100644 --- a/src/test/rbd_mirror/test_mock_ImageMap.cc +++ b/src/test/rbd_mirror/test_mock_ImageMap.cc @@ -135,24 +135,25 @@ public: : test_mock_image_map(test_mock_image_map) { } - MOCK_METHOD1(mock_acquire_image, void(const std::string &)); - MOCK_METHOD1(mock_release_image, void(const std::string &)); - MOCK_METHOD2(mock_remove_image, void(const std::string &, const std::string &)); + MOCK_METHOD2(mock_acquire_image, void(const std::string &, Context*)); + MOCK_METHOD2(mock_release_image, void(const std::string &, Context*)); + MOCK_METHOD3(mock_remove_image, void(const std::string &, + const std::string &, Context*)); void acquire_image(const std::string &global_image_id, - const std::string &instance_id) { - mock_acquire_image(global_image_id); + const std::string &instance_id, Context* on_finish) { + mock_acquire_image(global_image_id, on_finish); } void release_image(const std::string &global_image_id, - const std::string &instance_id) { - mock_release_image(global_image_id); + const std::string &instance_id, Context* on_finish) { + mock_release_image(global_image_id, on_finish); } void remove_image(const std::string &mirror_uuid, const std::string &global_image_id, - const std::string &instance_id) { - mock_remove_image(mirror_uuid, global_image_id); + const std::string &instance_id, Context* on_finish) { + mock_remove_image(mirror_uuid, global_image_id, on_finish); } }; @@ -200,20 +201,24 @@ public: } void expect_listener_acquire_image(MockListener &mock_listener, - const std::string &global_image_id) { - EXPECT_CALL(mock_listener, mock_acquire_image(global_image_id)) - .WillOnce(WithoutArgs(Invoke([this]() { + const std::string &global_image_id, + std::map *peer_ack_ctxs) { + EXPECT_CALL(mock_listener, mock_acquire_image(global_image_id, _)) + .WillOnce(WithArg<1>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) { Mutex::Locker locker(m_lock); + peer_ack_ctxs->insert({global_image_id, ctx}); ++m_notify_update_count; m_cond.Signal(); }))); } void expect_listener_release_image(MockListener &mock_listener, - const std::string &global_image_id) { - EXPECT_CALL(mock_listener, mock_release_image(global_image_id)) - .WillOnce(WithoutArgs(Invoke([this]() { + const std::string &global_image_id, + std::map *peer_ack_ctxs) { + EXPECT_CALL(mock_listener, mock_release_image(global_image_id, _)) + .WillOnce(WithArg<1>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) { Mutex::Locker locker(m_lock); + peer_ack_ctxs->insert({global_image_id, ctx}); ++m_notify_update_count; m_cond.Signal(); }))); @@ -221,46 +226,66 @@ public: void expect_listener_remove_image(MockListener &mock_listener, const std::string &mirror_uuid, - const std::string &global_image_id) { - EXPECT_CALL(mock_listener, mock_remove_image(mirror_uuid, global_image_id)) - .WillOnce(WithoutArgs(Invoke([this]() { + const std::string &global_image_id, + std::map *peer_ack_ctxs) { + EXPECT_CALL(mock_listener, + mock_remove_image(mirror_uuid, global_image_id, _)) + .WillOnce(WithArg<2>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) { Mutex::Locker locker(m_lock); + peer_ack_ctxs->insert({global_image_id, ctx}); ++m_notify_update_count; m_cond.Signal(); }))); } void expect_listener_images_unmapped(MockListener &mock_listener, - std::set *global_image_ids) { - EXPECT_CALL(mock_listener, mock_release_image(_)) + std::set *global_image_ids, + std::map *peer_ack_ctxs) { + EXPECT_CALL(mock_listener, mock_release_image(_, _)) .Times(AtLeast(0)) - .WillRepeatedly(WithArg<0>(Invoke([this, global_image_ids](std::string global_image_id) { + .WillRepeatedly(Invoke([this, global_image_ids, peer_ack_ctxs](std::string global_image_id, Context* ctx) { Mutex::Locker locker(m_lock); global_image_ids->emplace(global_image_id); + peer_ack_ctxs->insert({global_image_id, ctx}); ++m_notify_update_count; m_cond.Signal(); - }))); + })); } void remote_peer_ack_nowait(MockImageMap *image_map, - const std::set &global_image_ids, int ret) { - for (auto const &global_image_id : global_image_ids) { - image_map->handle_peer_ack(global_image_id, ret); + const std::set &global_image_ids, + int ret, + std::map *peer_ack_ctxs) { + for (auto& global_image_id : global_image_ids) { + auto it = peer_ack_ctxs->find(global_image_id); + ASSERT_TRUE(it != peer_ack_ctxs->end()); + it->second->complete(ret); + peer_ack_ctxs->erase(it); } } void remote_peer_ack_wait(MockImageMap *image_map, - const std::set &global_image_ids, int ret) { - for (auto const &global_image_id : global_image_ids) { - image_map->handle_peer_ack(global_image_id, ret); + const std::set &global_image_ids, + int ret, + std::map *peer_ack_ctxs) { + for (auto& global_image_id : global_image_ids) { + auto it = peer_ack_ctxs->find(global_image_id); + ASSERT_TRUE(it != peer_ack_ctxs->end()); + it->second->complete(ret); + peer_ack_ctxs->erase(it); ASSERT_TRUE(wait_for_map_update(1)); } } void remote_peer_ack_listener_wait(MockImageMap *image_map, - const std::set &global_image_ids, int ret) { - for (auto const &global_image_id : global_image_ids) { - image_map->handle_peer_ack(global_image_id, ret); + const std::set &global_image_ids, + int ret, + std::map *peer_ack_ctxs) { + for (auto& global_image_id : global_image_ids) { + auto it = peer_ack_ctxs->find(global_image_id); + ASSERT_TRUE(it != peer_ack_ctxs->end()); + it->second->complete(ret); + peer_ack_ctxs->erase(it); ASSERT_TRUE(wait_for_map_update(1)); ASSERT_TRUE(wait_for_listener_notify(1)); } @@ -269,12 +294,15 @@ public: void update_map_and_acquire(MockThreads &mock_threads, MockUpdateRequest &mock_update_request, MockListener &mock_listener, - const std::set &global_image_ids, int ret) { + const std::set &global_image_ids, + int ret, + std::map *peer_ack_ctxs) { for (auto const &global_image_id : global_image_ids) { expect_add_event(mock_threads); expect_update_request(mock_update_request, ret); expect_add_event(mock_threads); - expect_listener_acquire_image(mock_listener, global_image_id); + expect_listener_acquire_image(mock_listener, global_image_id, + peer_ack_ctxs); } } @@ -330,24 +358,30 @@ public: } void listener_acquire_images(MockListener &mock_listener, - const std::set &global_image_ids) { + const std::set &global_image_ids, + std::map *peer_ack_ctxs) { for (auto const &global_image_id : global_image_ids) { - expect_listener_acquire_image(mock_listener, global_image_id); + expect_listener_acquire_image(mock_listener, global_image_id, + peer_ack_ctxs); } } void listener_release_images(MockListener &mock_listener, - const std::set &global_image_ids) { + const std::set &global_image_ids, + std::map *peer_ack_ctxs) { for (auto const &global_image_id : global_image_ids) { - expect_listener_release_image(mock_listener, global_image_id); + expect_listener_release_image(mock_listener, global_image_id, + peer_ack_ctxs); } } void listener_remove_images(MockListener &mock_listener, const std::string &mirror_uuid, - std::set &global_image_ids) { + std::set &global_image_ids, + std::map *peer_ack_ctxs) { for (auto const &global_image_id : global_image_ids) { - expect_listener_remove_image(mock_listener, mirror_uuid, global_image_id); + expect_listener_remove_image(mock_listener, mirror_uuid, global_image_id, + peer_ack_ctxs); } } @@ -385,7 +419,8 @@ TEST_F(TestMockImageMap, SetLocalImages) { MockUpdateRequest mock_update_request; expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, global_image_ids); + std::map peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, &peer_ack_ctxs); // initial image list mock_image_map->update_images("", std::move(global_image_ids), {}); @@ -394,7 +429,8 @@ TEST_F(TestMockImageMap, SetLocalImages) { ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); // remote peer ACKs image acquire request - remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); wait_for_scheduled_task(); ASSERT_EQ(0, when_shut_down(mock_image_map.get())); @@ -433,7 +469,9 @@ TEST_F(TestMockImageMap, AddRemoveLocalImage) { MockUpdateRequest mock_update_request; expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, initial_global_image_ids); + std::map peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_global_image_ids, + &peer_ack_ctxs); // initial image list mock_image_map->update_images("", std::move(initial_global_image_ids), {}); @@ -442,18 +480,22 @@ TEST_F(TestMockImageMap, AddRemoveLocalImage) { ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size())); // remote peer ACKs image acquire request - remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0, + &peer_ack_ctxs); // RELEASE+REMOVE_MAPPING expect_add_event(mock_threads); - listener_release_images(mock_listener, remove_global_image_ids); - update_map_request(mock_threads, mock_update_request, remove_global_image_ids, 0); + listener_release_images(mock_listener, remove_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, remove_global_image_ids, + 0); // remove images mock_image_map->update_images("", {}, std::move(remove_global_image_ids)); ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size())); - remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0); + remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_ack_ctxs); wait_for_scheduled_task(); ASSERT_EQ(0, when_shut_down(mock_image_map.get())); @@ -492,28 +534,36 @@ TEST_F(TestMockImageMap, AddRemoveRemoteImage) { MockUpdateRequest mock_update_request; expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, initial_global_image_ids); + std::map peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_global_image_ids, + &peer_ack_ctxs); // initial image list - mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {}); + mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), + {}); ASSERT_TRUE(wait_for_map_update(1)); ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size())); // remote peer ACKs image acquire request - remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0, + &peer_ack_ctxs); // RELEASE+REMOVE_MAPPING - listener_remove_images(mock_listener, "uuid1", remove_global_image_ids); + listener_remove_images(mock_listener, "uuid1", remove_global_image_ids, + &peer_ack_ctxs); expect_add_event(mock_threads); - listener_release_images(mock_listener, remove_global_image_ids); - update_map_request(mock_threads, mock_update_request, remove_global_image_ids, 0); + listener_release_images(mock_listener, remove_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, remove_global_image_ids, + 0); // remove images mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids)); ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size() * 2)); - remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0); + remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_ack_ctxs); wait_for_scheduled_task(); ASSERT_EQ(0, when_shut_down(mock_image_map.get())); @@ -554,7 +604,9 @@ TEST_F(TestMockImageMap, AddRemoveRemoteImageDuplicateNotification) { MockUpdateRequest mock_update_request; expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, initial_global_image_ids); + std::map peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_global_image_ids, + &peer_ack_ctxs); // initial image list mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {}); @@ -567,19 +619,23 @@ TEST_F(TestMockImageMap, AddRemoveRemoteImageDuplicateNotification) { mock_image_map->update_images("uuid1", std::move(initial_global_image_ids_dup), {}); // remote peer ACKs image acquire request - remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0, + &peer_ack_ctxs); // RELEASE+REMOVE_MAPPING - listener_remove_images(mock_listener, "uuid1", remove_global_image_ids); + listener_remove_images(mock_listener, "uuid1", remove_global_image_ids, + &peer_ack_ctxs); expect_add_event(mock_threads); - listener_release_images(mock_listener, remove_global_image_ids); + listener_release_images(mock_listener, remove_global_image_ids, + &peer_ack_ctxs); update_map_request(mock_threads, mock_update_request, remove_global_image_ids, 0); // remove images mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids)); ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size() * 2)); - remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0); + remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_ack_ctxs); // trigger duplicate "remove" notification mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids_dup)); @@ -620,7 +676,9 @@ TEST_F(TestMockImageMap, AcquireImageErrorRetry) { expect_add_event(mock_threads); expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, initial_global_image_ids); + std::map peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_global_image_ids, + &peer_ack_ctxs); // initial image list mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {}); @@ -629,7 +687,8 @@ TEST_F(TestMockImageMap, AcquireImageErrorRetry) { ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size())); // remote peer ACKs image acquire request - remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0, + &peer_ack_ctxs); wait_for_scheduled_task(); ASSERT_EQ(0, when_shut_down(mock_image_map.get())); @@ -680,7 +739,9 @@ TEST_F(TestMockImageMap, RemoveRemoteAndLocalImage) { MockUpdateRequest mock_update_request; expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, initial_remote_global_image_ids); + std::map peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_remote_global_image_ids, + &peer_ack_ctxs); // initial remote image list mock_image_map->update_images("uuid1", std::move(initial_remote_global_image_ids), {}); @@ -689,28 +750,33 @@ TEST_F(TestMockImageMap, RemoveRemoteAndLocalImage) { ASSERT_TRUE(wait_for_listener_notify(initial_remote_global_image_ids_ack.size())); // remote peer ACKs image acquire request - remote_peer_ack_nowait(mock_image_map.get(), initial_remote_global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), + initial_remote_global_image_ids_ack, 0, + &peer_ack_ctxs); // set initial local image list -- this is a no-op from policy pov mock_image_map->update_images("", std::move(initial_local_global_image_ids), {}); // remove remote images -- this should be a no-op from policy pov // except the listener notification - listener_remove_images(mock_listener, "uuid1", remote_remove_global_image_ids); + listener_remove_images(mock_listener, "uuid1", remote_remove_global_image_ids, + &peer_ack_ctxs); mock_image_map->update_images("uuid1", {}, std::move(remote_remove_global_image_ids)); ASSERT_TRUE(wait_for_listener_notify(remote_remove_global_image_ids_ack.size())); // RELEASE+REMOVE_MAPPING expect_add_event(mock_threads); - listener_release_images(mock_listener, local_remove_global_image_ids); + listener_release_images(mock_listener, local_remove_global_image_ids, + &peer_ack_ctxs); update_map_request(mock_threads, mock_update_request, local_remove_global_image_ids, 0); // remove local images mock_image_map->update_images("", {}, std::move(local_remove_global_image_ids)); ASSERT_TRUE(wait_for_listener_notify(local_remove_global_image_ids_ack.size())); - remote_peer_ack_wait(mock_image_map.get(), local_remove_global_image_ids_ack, 0); + remote_peer_ack_wait(mock_image_map.get(), local_remove_global_image_ids_ack, + 0, &peer_ack_ctxs); wait_for_scheduled_task(); ASSERT_EQ(0, when_shut_down(mock_image_map.get())); @@ -744,7 +810,9 @@ TEST_F(TestMockImageMap, AddInstance) { MockUpdateRequest mock_update_request; expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, global_image_ids); + std::map peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); // initial image list mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); @@ -753,13 +821,15 @@ TEST_F(TestMockImageMap, AddInstance) { ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); // remote peer ACKs image acquire request - remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); std::set shuffled_global_image_ids; // RELEASE+UPDATE_MAPPING+ACQUIRE expect_add_event(mock_threads); - expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids); + expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids, + &peer_ack_ctxs); mock_image_map->update_instances_added({"9876"}); @@ -767,11 +837,14 @@ TEST_F(TestMockImageMap, AddInstance) { ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); update_map_and_acquire(mock_threads, mock_update_request, - mock_listener, shuffled_global_image_ids, 0); - remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0); + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); // completion shuffle action for now (re)mapped images - remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0); + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); wait_for_scheduled_task(); ASSERT_EQ(0, when_shut_down(mock_image_map.get())); @@ -806,7 +879,9 @@ TEST_F(TestMockImageMap, RemoveInstance) { MockUpdateRequest mock_update_request; expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, global_image_ids); + std::map peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); // set initial image list mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); @@ -815,31 +890,37 @@ TEST_F(TestMockImageMap, RemoveInstance) { ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); // remote peer ACKs image acquire request -- completing action - remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); std::set shuffled_global_image_ids; // RELEASE+UPDATE_MAPPING+ACQUIRE expect_add_event(mock_threads); - expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids); - + expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids, + &peer_ack_ctxs); + mock_image_map->update_instances_added({"9876"}); wait_for_scheduled_task(); ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); update_map_and_acquire(mock_threads, mock_update_request, - mock_listener, shuffled_global_image_ids, 0); - remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0); + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); // completion shuffle action for now (re)mapped images - remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0); + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); shuffled_global_image_ids.clear(); // remove added instance expect_add_event(mock_threads); - expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids); + expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids, + &peer_ack_ctxs); mock_image_map->update_instances_removed({"9876"}); @@ -847,11 +928,14 @@ TEST_F(TestMockImageMap, RemoveInstance) { ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); update_map_and_acquire(mock_threads, mock_update_request, - mock_listener, shuffled_global_image_ids, 0); - remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0); + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); // completion shuffle action for now (re)mapped images - remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0); + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); wait_for_scheduled_task(); ASSERT_EQ(0, when_shut_down(mock_image_map.get())); @@ -889,7 +973,9 @@ TEST_F(TestMockImageMap, AddInstancePingPongImageTest) { MockUpdateRequest mock_update_request; expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, global_image_ids); + std::map peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); // set initial image list mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); @@ -898,13 +984,15 @@ TEST_F(TestMockImageMap, AddInstancePingPongImageTest) { ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); // remote peer ACKs image acquire request -- completing action - remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); std::set shuffled_global_image_ids; // RELEASE+UPDATE_MAPPING+ACQUIRE expect_add_event(mock_threads); - expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids); + expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids, + &peer_ack_ctxs); mock_image_map->update_instances_added({"9876"}); @@ -912,18 +1000,22 @@ TEST_F(TestMockImageMap, AddInstancePingPongImageTest) { ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); update_map_and_acquire(mock_threads, mock_update_request, - mock_listener, shuffled_global_image_ids, 0); - remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0); + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); // completion shuffle action for now (re)mapped images - remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0); + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); std::set migrated_global_image_ids(shuffled_global_image_ids); shuffled_global_image_ids.clear(); // RELEASE+UPDATE_MAPPING+ACQUIRE expect_add_event(mock_threads); - expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids); + expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids, + &peer_ack_ctxs); // add another instance mock_image_map->update_instances_added({"5432"}); @@ -932,11 +1024,14 @@ TEST_F(TestMockImageMap, AddInstancePingPongImageTest) { ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); update_map_and_acquire(mock_threads, mock_update_request, - mock_listener, shuffled_global_image_ids, 0); - remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0); + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); // completion shuffle action for now (re)mapped images - remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0); + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); // shuffle set should be distinct std::set reshuffled; @@ -982,7 +1077,9 @@ TEST_F(TestMockImageMap, RemoveInstanceWithRemoveImage) { MockUpdateRequest mock_update_request; expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, global_image_ids); + std::map peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); // initial image list mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); @@ -990,13 +1087,15 @@ TEST_F(TestMockImageMap, RemoveInstanceWithRemoveImage) { ASSERT_TRUE(wait_for_map_update(1)); ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); - remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); std::set shuffled_global_image_ids; // RELEASE+UPDATE_MAPPING+ACQUIRE expect_add_event(mock_threads); - expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids); + expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids, + &peer_ack_ctxs); mock_image_map->update_instances_added({"9876"}); @@ -1004,18 +1103,23 @@ TEST_F(TestMockImageMap, RemoveInstanceWithRemoveImage) { ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); update_map_and_acquire(mock_threads, mock_update_request, - mock_listener, shuffled_global_image_ids, 0); - remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0); + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); // completion shuffle action for now (re)mapped images - remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0); + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); std::set shuffled_global_image_ids_ack(shuffled_global_image_ids); // RELEASE - listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids); + listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids, + &peer_ack_ctxs); expect_add_event(mock_threads); - listener_release_images(mock_listener, shuffled_global_image_ids); + listener_release_images(mock_listener, shuffled_global_image_ids, + &peer_ack_ctxs); mock_image_map->update_images("uuid1", {}, std::move(shuffled_global_image_ids)); ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size() * 2)); @@ -1023,7 +1127,8 @@ TEST_F(TestMockImageMap, RemoveInstanceWithRemoveImage) { // instance failed -- update policy for instance removal mock_image_map->update_instances_removed({"9876"}); - remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, -EBLACKLISTED); + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, + -EBLACKLISTED, &peer_ack_ctxs); wait_for_scheduled_task(); ASSERT_EQ(0, when_shut_down(mock_image_map.get())); @@ -1057,7 +1162,9 @@ TEST_F(TestMockImageMap, AddErrorAndRemoveImage) { MockUpdateRequest mock_update_request; expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, global_image_ids); + std::map peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); // initial image list mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); @@ -1066,13 +1173,15 @@ TEST_F(TestMockImageMap, AddErrorAndRemoveImage) { ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); // remote peer ACKs image acquire request - remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); std::set shuffled_global_image_ids; // RELEASE+UPDATE_MAPPING+ACQUIRE expect_add_event(mock_threads); - expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids); + expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids, + &peer_ack_ctxs); mock_image_map->update_instances_added({"9876"}); @@ -1080,32 +1189,40 @@ TEST_F(TestMockImageMap, AddErrorAndRemoveImage) { ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); update_map_and_acquire(mock_threads, mock_update_request, - mock_listener, shuffled_global_image_ids, 0); - remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0); + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); mock_image_map->update_instances_removed({"9876"}); // instance blacklisted -- ACQUIRE request fails update_map_and_acquire(mock_threads, mock_update_request, - mock_listener, shuffled_global_image_ids, 0); - remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, -EBLACKLISTED); + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + -EBLACKLISTED, &peer_ack_ctxs); // new peer acks acquire request - remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0); + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); wait_for_scheduled_task(); std::set shuffled_global_image_ids_ack(shuffled_global_image_ids); // remove image - listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids); + listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids, + &peer_ack_ctxs); expect_add_event(mock_threads); - listener_release_images(mock_listener, shuffled_global_image_ids); + listener_release_images(mock_listener, shuffled_global_image_ids, + &peer_ack_ctxs); update_map_request(mock_threads, mock_update_request, shuffled_global_image_ids, 0); mock_image_map->update_images("uuid1", {}, std::move(shuffled_global_image_ids)); ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size() * 2)); - remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids_ack, 0); + remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids_ack, 0, + &peer_ack_ctxs); wait_for_scheduled_task(); ASSERT_EQ(0, when_shut_down(mock_image_map.get())); @@ -1151,7 +1268,9 @@ TEST_F(TestMockImageMap, MirrorUUIDUpdated) { MockUpdateRequest mock_update_request; expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, initial_remote_global_image_ids); + std::map peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_remote_global_image_ids, + &peer_ack_ctxs); // initial remote image list mock_image_map->update_images("uuid1", std::move(initial_remote_global_image_ids), {}); @@ -1160,24 +1279,31 @@ TEST_F(TestMockImageMap, MirrorUUIDUpdated) { ASSERT_TRUE(wait_for_listener_notify(initial_remote_global_image_ids_ack.size())); // remote peer ACKs image acquire request - remote_peer_ack_nowait(mock_image_map.get(), initial_remote_global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), + initial_remote_global_image_ids_ack, 0, + &peer_ack_ctxs); // RELEASE+REMOVE_MAPPING - listener_remove_images(mock_listener, "uuid1", remote_removed_global_image_ids); + listener_remove_images(mock_listener, "uuid1", remote_removed_global_image_ids, + &peer_ack_ctxs); expect_add_event(mock_threads); - listener_release_images(mock_listener, remote_removed_global_image_ids); + listener_release_images(mock_listener, remote_removed_global_image_ids, + &peer_ack_ctxs); update_map_request(mock_threads, mock_update_request, remote_removed_global_image_ids, 0); mock_image_map->update_images("uuid1", {}, std::move(remote_removed_global_image_ids)); ASSERT_TRUE(wait_for_listener_notify(remote_removed_global_image_ids_ack.size() * 2)); - remote_peer_ack_wait(mock_image_map.get(), remote_removed_global_image_ids_ack, 0); + remote_peer_ack_wait(mock_image_map.get(), + remote_removed_global_image_ids_ack, 0, + &peer_ack_ctxs); // UPDATE_MAPPING+ACQUIRE expect_add_event(mock_threads); expect_update_request(mock_update_request, 0); expect_add_event(mock_threads); - listener_acquire_images(mock_listener, remote_added_global_image_ids); + listener_acquire_images(mock_listener, remote_added_global_image_ids, + &peer_ack_ctxs); mock_image_map->update_images("uuid2", std::move(remote_added_global_image_ids), {}); @@ -1185,7 +1311,9 @@ TEST_F(TestMockImageMap, MirrorUUIDUpdated) { ASSERT_TRUE(wait_for_listener_notify(remote_added_global_image_ids_ack.size())); // remote peer ACKs image acquire request - remote_peer_ack_nowait(mock_image_map.get(), remote_added_global_image_ids_ack, 0); + remote_peer_ack_nowait(mock_image_map.get(), + remote_added_global_image_ids_ack, 0, + &peer_ack_ctxs); wait_for_scheduled_task(); ASSERT_EQ(0, when_shut_down(mock_image_map.get())); diff --git a/src/tools/rbd_mirror/ImageMap.cc b/src/tools/rbd_mirror/ImageMap.cc index 261d02f6c1756..b945e2c9ac876 100644 --- a/src/tools/rbd_mirror/ImageMap.cc +++ b/src/tools/rbd_mirror/ImageMap.cc @@ -27,6 +27,22 @@ using image_map::Policy; using librbd::util::unique_lock_name; using librbd::util::create_async_context_callback; +template +struct ImageMap::C_NotifyInstance : public Context { + ImageMap* image_map; + std::string global_image_id; + + C_NotifyInstance(ImageMap* image_map, const std::string& global_image_id) + : image_map(image_map), global_image_id(global_image_id) { + image_map->start_async_op(); + } + + void finish(int r) override { + image_map->handle_peer_ack(global_image_id, r); + image_map->finish_async_op(); + } +}; + template ImageMap::ImageMap(librados::IoCtx &ioctx, Threads *threads, image_map::Listener &listener) : m_ioctx(ioctx), @@ -251,17 +267,25 @@ void ImageMap::schedule_action(const std::string &global_image_id) { } template -void ImageMap::notify_listener_acquire_release_images(const Updates &acquire, - const Updates &release) { +void ImageMap::notify_listener_acquire_release_images( + const Updates &acquire, const Updates &release) { dout(20) << ": acquire_count: " << acquire.size() << ", release_count=" << release.size() << dendl; for (auto const &update : acquire) { - m_listener.acquire_image(update.global_image_id, update.instance_id); + m_listener.acquire_image( + update.global_image_id, update.instance_id, + create_async_context_callback( + m_threads->work_queue, + new C_NotifyInstance(this, update.global_image_id))); } for (auto const &update : release) { - m_listener.release_image(update.global_image_id, update.instance_id); + m_listener.release_image( + update.global_image_id, update.instance_id, + create_async_context_callback( + m_threads->work_queue, + new C_NotifyInstance(this, update.global_image_id))); } } @@ -272,7 +296,11 @@ void ImageMap::notify_listener_remove_images(const std::string &peer_uuid, << dendl; for (auto const &update : remove) { - m_listener.remove_image(peer_uuid, update.global_image_id, update.instance_id); + m_listener.remove_image( + peer_uuid, update.global_image_id, update.instance_id, + create_async_context_callback( + m_threads->work_queue, + new C_NotifyInstance(this, update.global_image_id))); } } diff --git a/src/tools/rbd_mirror/ImageMap.h b/src/tools/rbd_mirror/ImageMap.h index a10b840605131..ffe91742bb31a 100644 --- a/src/tools/rbd_mirror/ImageMap.h +++ b/src/tools/rbd_mirror/ImageMap.h @@ -43,14 +43,13 @@ public: std::set &&added_global_image_ids, std::set &&removed_global_image_ids); - // handle notify response from remote peer (r : 0 == success, negative otherwise) - void handle_peer_ack(const std::string &global_image_id, int r); - // add/remove instances void update_instances_added(const std::vector &instances); void update_instances_removed(const std::vector &instances); private: + struct C_NotifyInstance; + ImageMap(librados::IoCtx &ioctx, Threads *threads, image_map::Listener &listener); @@ -188,6 +187,8 @@ private: bool add_peer(const std::string &global_image_id, const std::string &peer_uuid); bool remove_peer(const std::string &global_image_id, const std::string &peer_uuid); + void handle_peer_ack(const std::string &global_image_id, int r); + // queue on-disk,acquire,remove updates in appropriate list void queue_update_map(const std::string &global_image_id); void queue_remove_map(const std::string &global_image_id); diff --git a/src/tools/rbd_mirror/image_map/Types.h b/src/tools/rbd_mirror/image_map/Types.h index 32fe6c24011f0..56433fa1b1cd2 100644 --- a/src/tools/rbd_mirror/image_map/Types.h +++ b/src/tools/rbd_mirror/image_map/Types.h @@ -13,6 +13,8 @@ #include "include/utime.h" #include "tools/rbd_mirror/types.h" +struct Context; + namespace ceph { class Formatter; } @@ -28,12 +30,15 @@ struct Listener { } virtual void acquire_image(const std::string &global_image_id, - const std::string &instance_id) = 0; + const std::string &instance_id, + Context* on_finish) = 0; virtual void release_image(const std::string &global_image_id, - const std::string &instance_id) = 0; + const std::string &instance_id, + Context* on_finish) = 0; virtual void remove_image(const std::string &mirror_uuid, const std::string &global_image_id, - const std::string &instance_id) = 0; + const std::string &instance_id, + Context* on_finish) = 0; }; struct LookupInfo { -- 2.39.5