rbd-mirror: image map listener should provide a callback context

The context wraps the necessary data and provides a pass-through chain
to the instance notification helpers.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/test/rbd_mirror/test_mock_ImageMap.cc b/src/test/rbd_mirror/test_mock_ImageMap.cc
index 090fb94f74c226752f9593152a68d60df92f2845..f9aa231ed247fec81e82e036c45515e6edfed4fb 100644 (file)
--- a/src/test/rbd_mirror/test_mock_ImageMap.cc
+++ b/src/test/rbd_mirror/test_mock_ImageMap.cc
@@ -135,24 +135,25 @@ public:
       : test_mock_image_map(test_mock_image_map) {
     }
 
-    MOCK_METHOD1(mock_acquire_image, void(const std::string &));
-    MOCK_METHOD1(mock_release_image, void(const std::string &));
-    MOCK_METHOD2(mock_remove_image, void(const std::string &, const std::string &));
+    MOCK_METHOD2(mock_acquire_image, void(const std::string &, Context*));
+    MOCK_METHOD2(mock_release_image, void(const std::string &, Context*));
+    MOCK_METHOD3(mock_remove_image, void(const std::string &,
+                                         const std::string &, Context*));
 
     void acquire_image(const std::string &global_image_id,
-                       const std::string &instance_id) {
-      mock_acquire_image(global_image_id);
+                       const std::string &instance_id, Context* on_finish) {
+      mock_acquire_image(global_image_id, on_finish);
     }
 
     void release_image(const std::string &global_image_id,
-                       const std::string &instance_id) {
-      mock_release_image(global_image_id);
+                       const std::string &instance_id, Context* on_finish) {
+      mock_release_image(global_image_id, on_finish);
     }
 
     void remove_image(const std::string &mirror_uuid,
                       const std::string &global_image_id,
-                      const std::string &instance_id) {
-      mock_remove_image(mirror_uuid, global_image_id);
+                      const std::string &instance_id, Context* on_finish) {
+      mock_remove_image(mirror_uuid, global_image_id, on_finish);
     }
   };
 
@@ -200,20 +201,24 @@ public:
   }
 
   void expect_listener_acquire_image(MockListener &mock_listener,
-                                     const std::string &global_image_id) {
-    EXPECT_CALL(mock_listener, mock_acquire_image(global_image_id))
-      .WillOnce(WithoutArgs(Invoke([this]() {
+                                     const std::string &global_image_id,
+                                     std::map<std::string, Context*> *peer_ack_ctxs) {
+    EXPECT_CALL(mock_listener, mock_acquire_image(global_image_id, _))
+      .WillOnce(WithArg<1>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) {
               Mutex::Locker locker(m_lock);
+              peer_ack_ctxs->insert({global_image_id, ctx});
               ++m_notify_update_count;
               m_cond.Signal();
             })));
   }
 
   void expect_listener_release_image(MockListener &mock_listener,
-                                     const std::string &global_image_id) {
-    EXPECT_CALL(mock_listener, mock_release_image(global_image_id))
-      .WillOnce(WithoutArgs(Invoke([this]() {
+                                     const std::string &global_image_id,
+                                     std::map<std::string, Context*> *peer_ack_ctxs) {
+    EXPECT_CALL(mock_listener, mock_release_image(global_image_id, _))
+      .WillOnce(WithArg<1>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) {
               Mutex::Locker locker(m_lock);
+              peer_ack_ctxs->insert({global_image_id, ctx});
               ++m_notify_update_count;
               m_cond.Signal();
             })));
@@ -221,46 +226,66 @@ public:
 
   void expect_listener_remove_image(MockListener &mock_listener,
                                     const std::string &mirror_uuid,
-                                    const std::string &global_image_id) {
-    EXPECT_CALL(mock_listener, mock_remove_image(mirror_uuid, global_image_id))
-      .WillOnce(WithoutArgs(Invoke([this]() {
+                                    const std::string &global_image_id,
+                                    std::map<std::string, Context*> *peer_ack_ctxs) {
+    EXPECT_CALL(mock_listener,
+                mock_remove_image(mirror_uuid, global_image_id, _))
+      .WillOnce(WithArg<2>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) {
               Mutex::Locker locker(m_lock);
+              peer_ack_ctxs->insert({global_image_id, ctx});
               ++m_notify_update_count;
               m_cond.Signal();
             })));
   }
 
   void expect_listener_images_unmapped(MockListener &mock_listener,
-                                       std::set<std::string> *global_image_ids) {
-    EXPECT_CALL(mock_listener, mock_release_image(_))
+                                       std::set<std::string> *global_image_ids,
+                                       std::map<std::string, Context*> *peer_ack_ctxs) {
+    EXPECT_CALL(mock_listener, mock_release_image(_, _))
       .Times(AtLeast(0))
-      .WillRepeatedly(WithArg<0>(Invoke([this, global_image_ids](std::string global_image_id) {
+      .WillRepeatedly(Invoke([this, global_image_ids, peer_ack_ctxs](std::string global_image_id, Context* ctx) {
               Mutex::Locker locker(m_lock);
               global_image_ids->emplace(global_image_id);
+              peer_ack_ctxs->insert({global_image_id, ctx});
               ++m_notify_update_count;
               m_cond.Signal();
-            })));
+            }));
   }
 
   void remote_peer_ack_nowait(MockImageMap *image_map,
-                                const std::set<std::string> &global_image_ids, int ret) {
-    for (auto const &global_image_id : global_image_ids) {
-      image_map->handle_peer_ack(global_image_id, ret);
+                              const std::set<std::string> &global_image_ids,
+                              int ret,
+                              std::map<std::string, Context*> *peer_ack_ctxs) {
+    for (auto& global_image_id : global_image_ids) {
+      auto it = peer_ack_ctxs->find(global_image_id);
+      ASSERT_TRUE(it != peer_ack_ctxs->end());
+      it->second->complete(ret);
+      peer_ack_ctxs->erase(it);
     }
   }
 
   void remote_peer_ack_wait(MockImageMap *image_map,
-                            const std::set<std::string> &global_image_ids, int ret) {
-    for (auto const &global_image_id : global_image_ids) {
-      image_map->handle_peer_ack(global_image_id, ret);
+                            const std::set<std::string> &global_image_ids,
+                            int ret,
+                            std::map<std::string, Context*> *peer_ack_ctxs) {
+    for (auto& global_image_id : global_image_ids) {
+      auto it = peer_ack_ctxs->find(global_image_id);
+      ASSERT_TRUE(it != peer_ack_ctxs->end());
+      it->second->complete(ret);
+      peer_ack_ctxs->erase(it);
       ASSERT_TRUE(wait_for_map_update(1));
     }
   }
 
   void remote_peer_ack_listener_wait(MockImageMap *image_map,
-                                     const std::set<std::string> &global_image_ids, int ret) {
-    for (auto const &global_image_id : global_image_ids) {
-      image_map->handle_peer_ack(global_image_id, ret);
+                                     const std::set<std::string> &global_image_ids,
+                                     int ret,
+                                     std::map<std::string, Context*> *peer_ack_ctxs) {
+    for (auto& global_image_id : global_image_ids) {
+      auto it = peer_ack_ctxs->find(global_image_id);
+      ASSERT_TRUE(it != peer_ack_ctxs->end());
+      it->second->complete(ret);
+      peer_ack_ctxs->erase(it);
       ASSERT_TRUE(wait_for_map_update(1));
       ASSERT_TRUE(wait_for_listener_notify(1));
     }
@@ -269,12 +294,15 @@ public:
   void update_map_and_acquire(MockThreads &mock_threads,
                               MockUpdateRequest &mock_update_request,
                               MockListener &mock_listener,
-                              const std::set<std::string> &global_image_ids, int ret) {
+                              const std::set<std::string> &global_image_ids,
+                              int ret,
+                              std::map<std::string, Context*> *peer_ack_ctxs) {
     for (auto const &global_image_id : global_image_ids) {
       expect_add_event(mock_threads);
       expect_update_request(mock_update_request, ret);
       expect_add_event(mock_threads);
-      expect_listener_acquire_image(mock_listener, global_image_id);
+      expect_listener_acquire_image(mock_listener, global_image_id,
+                                    peer_ack_ctxs);
     }
   }
 
@@ -330,24 +358,30 @@ public:
   }
 
   void listener_acquire_images(MockListener &mock_listener,
-                               const std::set<std::string> &global_image_ids) {
+                               const std::set<std::string> &global_image_ids,
+                               std::map<std::string, Context*> *peer_ack_ctxs) {
     for (auto const &global_image_id : global_image_ids) {
-      expect_listener_acquire_image(mock_listener, global_image_id);
+      expect_listener_acquire_image(mock_listener, global_image_id,
+                                    peer_ack_ctxs);
     }
   }
 
   void listener_release_images(MockListener &mock_listener,
-                               const std::set<std::string> &global_image_ids) {
+                               const std::set<std::string> &global_image_ids,
+                               std::map<std::string, Context*> *peer_ack_ctxs) {
     for (auto const &global_image_id : global_image_ids) {
-      expect_listener_release_image(mock_listener, global_image_id);
+      expect_listener_release_image(mock_listener, global_image_id,
+                                    peer_ack_ctxs);
     }
   }
 
   void listener_remove_images(MockListener &mock_listener,
                               const std::string &mirror_uuid,
-                              std::set<std::string> &global_image_ids) {
+                              std::set<std::string> &global_image_ids,
+                              std::map<std::string, Context*> *peer_ack_ctxs) {
     for (auto const &global_image_id : global_image_ids) {
-      expect_listener_remove_image(mock_listener, mirror_uuid, global_image_id);
+      expect_listener_remove_image(mock_listener, mirror_uuid, global_image_id,
+                                   peer_ack_ctxs);
     }
   }
 
@@ -385,7 +419,8 @@ TEST_F(TestMockImageMap, SetLocalImages) {
   MockUpdateRequest mock_update_request;
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, global_image_ids);
+  std::map<std::string, Context*> peer_ack_ctxs;
+  listener_acquire_images(mock_listener, global_image_ids, &peer_ack_ctxs);
 
   // initial image list
   mock_image_map->update_images("", std::move(global_image_ids), {});
@@ -394,7 +429,8 @@ TEST_F(TestMockImageMap, SetLocalImages) {
   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
 
   // remote peer ACKs image acquire request
-  remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   wait_for_scheduled_task();
   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
@@ -433,7 +469,9 @@ TEST_F(TestMockImageMap, AddRemoveLocalImage) {
   MockUpdateRequest mock_update_request;
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, initial_global_image_ids);
+  std::map<std::string, Context*> peer_ack_ctxs;
+  listener_acquire_images(mock_listener, initial_global_image_ids,
+                          &peer_ack_ctxs);
 
   // initial image list
   mock_image_map->update_images("", std::move(initial_global_image_ids), {});
@@ -442,18 +480,22 @@ TEST_F(TestMockImageMap, AddRemoveLocalImage) {
   ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size()));
 
   // remote peer ACKs image acquire request
-  remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   // RELEASE+REMOVE_MAPPING
   expect_add_event(mock_threads);
-  listener_release_images(mock_listener, remove_global_image_ids);
-  update_map_request(mock_threads, mock_update_request, remove_global_image_ids, 0);
+  listener_release_images(mock_listener, remove_global_image_ids,
+                          &peer_ack_ctxs);
+  update_map_request(mock_threads, mock_update_request, remove_global_image_ids,
+                     0);
 
   // remove images
   mock_image_map->update_images("", {}, std::move(remove_global_image_ids));
   ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size()));
 
-  remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0);
+  remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0,
+                       &peer_ack_ctxs);
 
   wait_for_scheduled_task();
   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
@@ -492,28 +534,36 @@ TEST_F(TestMockImageMap, AddRemoveRemoteImage) {
   MockUpdateRequest mock_update_request;
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, initial_global_image_ids);
+  std::map<std::string, Context*> peer_ack_ctxs;
+  listener_acquire_images(mock_listener, initial_global_image_ids,
+                          &peer_ack_ctxs);
 
   // initial image list
-  mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {});
+  mock_image_map->update_images("uuid1", std::move(initial_global_image_ids),
+                                {});
 
   ASSERT_TRUE(wait_for_map_update(1));
   ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size()));
 
   // remote peer ACKs image acquire request
-  remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   // RELEASE+REMOVE_MAPPING
-  listener_remove_images(mock_listener, "uuid1", remove_global_image_ids);
+  listener_remove_images(mock_listener, "uuid1", remove_global_image_ids,
+                         &peer_ack_ctxs);
   expect_add_event(mock_threads);
-  listener_release_images(mock_listener, remove_global_image_ids);
-  update_map_request(mock_threads, mock_update_request, remove_global_image_ids, 0);
+  listener_release_images(mock_listener, remove_global_image_ids,
+                          &peer_ack_ctxs);
+  update_map_request(mock_threads, mock_update_request, remove_global_image_ids,
+                     0);
 
   // remove images
   mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids));
   ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size() * 2));
 
-  remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0);
+  remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0,
+                       &peer_ack_ctxs);
 
   wait_for_scheduled_task();
   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
@@ -554,7 +604,9 @@ TEST_F(TestMockImageMap, AddRemoveRemoteImageDuplicateNotification) {
   MockUpdateRequest mock_update_request;
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, initial_global_image_ids);
+  std::map<std::string, Context*> peer_ack_ctxs;
+  listener_acquire_images(mock_listener, initial_global_image_ids,
+                          &peer_ack_ctxs);
 
   // initial image list
   mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {});
@@ -567,19 +619,23 @@ TEST_F(TestMockImageMap, AddRemoveRemoteImageDuplicateNotification) {
   mock_image_map->update_images("uuid1", std::move(initial_global_image_ids_dup), {});
 
   // remote peer ACKs image acquire request
-  remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   // RELEASE+REMOVE_MAPPING
-  listener_remove_images(mock_listener, "uuid1", remove_global_image_ids);
+  listener_remove_images(mock_listener, "uuid1", remove_global_image_ids,
+                         &peer_ack_ctxs);
   expect_add_event(mock_threads);
-  listener_release_images(mock_listener, remove_global_image_ids);
+  listener_release_images(mock_listener, remove_global_image_ids,
+                          &peer_ack_ctxs);
   update_map_request(mock_threads, mock_update_request, remove_global_image_ids, 0);
 
   // remove images
   mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids));
   ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size() * 2));
 
-  remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0);
+  remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0,
+                       &peer_ack_ctxs);
 
   // trigger duplicate "remove" notification
   mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids_dup));
@@ -620,7 +676,9 @@ TEST_F(TestMockImageMap, AcquireImageErrorRetry) {
   expect_add_event(mock_threads);
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, initial_global_image_ids);
+  std::map<std::string, Context*> peer_ack_ctxs;
+  listener_acquire_images(mock_listener, initial_global_image_ids,
+                          &peer_ack_ctxs);
 
   // initial image list
   mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {});
@@ -629,7 +687,8 @@ TEST_F(TestMockImageMap, AcquireImageErrorRetry) {
   ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size()));
 
   // remote peer ACKs image acquire request
-  remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   wait_for_scheduled_task();
   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
@@ -680,7 +739,9 @@ TEST_F(TestMockImageMap, RemoveRemoteAndLocalImage) {
   MockUpdateRequest mock_update_request;
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, initial_remote_global_image_ids);
+  std::map<std::string, Context*> peer_ack_ctxs;
+  listener_acquire_images(mock_listener, initial_remote_global_image_ids,
+                          &peer_ack_ctxs);
 
   // initial remote image list
   mock_image_map->update_images("uuid1", std::move(initial_remote_global_image_ids), {});
@@ -689,28 +750,33 @@ TEST_F(TestMockImageMap, RemoveRemoteAndLocalImage) {
   ASSERT_TRUE(wait_for_listener_notify(initial_remote_global_image_ids_ack.size()));
 
   // remote peer ACKs image acquire request
-  remote_peer_ack_nowait(mock_image_map.get(), initial_remote_global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(),
+                         initial_remote_global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   // set initial local image list -- this is a no-op from policy pov
   mock_image_map->update_images("", std::move(initial_local_global_image_ids), {});
 
   // remove remote images -- this should be a no-op from policy pov
   // except the listener notification
-  listener_remove_images(mock_listener, "uuid1", remote_remove_global_image_ids);
+  listener_remove_images(mock_listener, "uuid1", remote_remove_global_image_ids,
+                         &peer_ack_ctxs);
 
   mock_image_map->update_images("uuid1", {}, std::move(remote_remove_global_image_ids));
   ASSERT_TRUE(wait_for_listener_notify(remote_remove_global_image_ids_ack.size()));
 
   // RELEASE+REMOVE_MAPPING
   expect_add_event(mock_threads);
-  listener_release_images(mock_listener, local_remove_global_image_ids);
+  listener_release_images(mock_listener, local_remove_global_image_ids,
+                          &peer_ack_ctxs);
   update_map_request(mock_threads, mock_update_request, local_remove_global_image_ids, 0);
 
   // remove local images
   mock_image_map->update_images("", {}, std::move(local_remove_global_image_ids));
   ASSERT_TRUE(wait_for_listener_notify(local_remove_global_image_ids_ack.size()));
 
-  remote_peer_ack_wait(mock_image_map.get(), local_remove_global_image_ids_ack, 0);
+  remote_peer_ack_wait(mock_image_map.get(), local_remove_global_image_ids_ack,
+                       0, &peer_ack_ctxs);
 
   wait_for_scheduled_task();
   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
@@ -744,7 +810,9 @@ TEST_F(TestMockImageMap, AddInstance) {
   MockUpdateRequest mock_update_request;
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, global_image_ids);
+  std::map<std::string, Context*> peer_ack_ctxs;
+  listener_acquire_images(mock_listener, global_image_ids,
+                          &peer_ack_ctxs);
 
   // initial image list
   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
@@ -753,13 +821,15 @@ TEST_F(TestMockImageMap, AddInstance) {
   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
 
   // remote peer ACKs image acquire request
-  remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   std::set<std::string> shuffled_global_image_ids;
 
   // RELEASE+UPDATE_MAPPING+ACQUIRE
   expect_add_event(mock_threads);
-  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids);
+  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids,
+                                  &peer_ack_ctxs);
 
   mock_image_map->update_instances_added({"9876"});
 
@@ -767,11 +837,14 @@ TEST_F(TestMockImageMap, AddInstance) {
   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 
   update_map_and_acquire(mock_threads, mock_update_request,
-                         mock_listener, shuffled_global_image_ids, 0);
-  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0);
+                         mock_listener, shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
+  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
+                                0, &peer_ack_ctxs);
 
   // completion shuffle action for now (re)mapped images
-  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
 
   wait_for_scheduled_task();
   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
@@ -806,7 +879,9 @@ TEST_F(TestMockImageMap, RemoveInstance) {
   MockUpdateRequest mock_update_request;
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, global_image_ids);
+  std::map<std::string, Context*> peer_ack_ctxs;
+  listener_acquire_images(mock_listener, global_image_ids,
+                          &peer_ack_ctxs);
 
   // set initial image list
   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
@@ -815,31 +890,37 @@ TEST_F(TestMockImageMap, RemoveInstance) {
   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
 
   // remote peer ACKs image acquire request -- completing action
-  remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   std::set<std::string> shuffled_global_image_ids;
 
   // RELEASE+UPDATE_MAPPING+ACQUIRE
   expect_add_event(mock_threads);
-  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids);
-  
+  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids,
+                                  &peer_ack_ctxs);
+
   mock_image_map->update_instances_added({"9876"});
 
   wait_for_scheduled_task();
   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 
   update_map_and_acquire(mock_threads, mock_update_request,
-                         mock_listener, shuffled_global_image_ids, 0);
-  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0);
+                         mock_listener, shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
+  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
+                                0, &peer_ack_ctxs);
 
   // completion shuffle action for now (re)mapped images
-  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
 
   shuffled_global_image_ids.clear();
 
   // remove added instance
   expect_add_event(mock_threads);
-  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids);
+  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids,
+                                  &peer_ack_ctxs);
 
   mock_image_map->update_instances_removed({"9876"});
 
@@ -847,11 +928,14 @@ TEST_F(TestMockImageMap, RemoveInstance) {
   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 
   update_map_and_acquire(mock_threads, mock_update_request,
-                         mock_listener, shuffled_global_image_ids, 0);
-  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0);
+                         mock_listener, shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
+  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
+                                0, &peer_ack_ctxs);
 
   // completion shuffle action for now (re)mapped images
-  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
 
   wait_for_scheduled_task();
   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
@@ -889,7 +973,9 @@ TEST_F(TestMockImageMap, AddInstancePingPongImageTest) {
   MockUpdateRequest mock_update_request;
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, global_image_ids);
+  std::map<std::string, Context*> peer_ack_ctxs;
+  listener_acquire_images(mock_listener, global_image_ids,
+                          &peer_ack_ctxs);
 
   // set initial image list
   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
@@ -898,13 +984,15 @@ TEST_F(TestMockImageMap, AddInstancePingPongImageTest) {
   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
 
   // remote peer ACKs image acquire request -- completing action
-  remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   std::set<std::string> shuffled_global_image_ids;
 
   // RELEASE+UPDATE_MAPPING+ACQUIRE
   expect_add_event(mock_threads);
-  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids);
+  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids,
+                                  &peer_ack_ctxs);
 
   mock_image_map->update_instances_added({"9876"});
 
@@ -912,18 +1000,22 @@ TEST_F(TestMockImageMap, AddInstancePingPongImageTest) {
   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 
   update_map_and_acquire(mock_threads, mock_update_request,
-                         mock_listener, shuffled_global_image_ids, 0);
-  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0);
+                         mock_listener, shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
+  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
+                                0, &peer_ack_ctxs);
 
   // completion shuffle action for now (re)mapped images
-  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
 
   std::set<std::string> migrated_global_image_ids(shuffled_global_image_ids);
   shuffled_global_image_ids.clear();
 
   // RELEASE+UPDATE_MAPPING+ACQUIRE
   expect_add_event(mock_threads);
-  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids);
+  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids,
+                                  &peer_ack_ctxs);
 
   // add another instance
   mock_image_map->update_instances_added({"5432"});
@@ -932,11 +1024,14 @@ TEST_F(TestMockImageMap, AddInstancePingPongImageTest) {
   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 
   update_map_and_acquire(mock_threads, mock_update_request,
-                         mock_listener, shuffled_global_image_ids, 0);
-  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0);
+                         mock_listener, shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
+  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
+                                0, &peer_ack_ctxs);
 
   // completion shuffle action for now (re)mapped images
-  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
 
   // shuffle set should be distinct
   std::set<std::string> reshuffled;
@@ -982,7 +1077,9 @@ TEST_F(TestMockImageMap, RemoveInstanceWithRemoveImage) {
   MockUpdateRequest mock_update_request;
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, global_image_ids);
+  std::map<std::string, Context*> peer_ack_ctxs;
+  listener_acquire_images(mock_listener, global_image_ids,
+                          &peer_ack_ctxs);
 
   // initial image list
   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
@@ -990,13 +1087,15 @@ TEST_F(TestMockImageMap, RemoveInstanceWithRemoveImage) {
   ASSERT_TRUE(wait_for_map_update(1));
   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
 
-  remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   std::set<std::string> shuffled_global_image_ids;
 
   // RELEASE+UPDATE_MAPPING+ACQUIRE
   expect_add_event(mock_threads);
-  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids);
+  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids,
+                                  &peer_ack_ctxs);
 
   mock_image_map->update_instances_added({"9876"});
 
@@ -1004,18 +1103,23 @@ TEST_F(TestMockImageMap, RemoveInstanceWithRemoveImage) {
   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 
   update_map_and_acquire(mock_threads, mock_update_request,
-                         mock_listener, shuffled_global_image_ids, 0);
-  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0);
+                         mock_listener, shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
+  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
+                                0, &peer_ack_ctxs);
 
   // completion shuffle action for now (re)mapped images
-  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
 
   std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids);
 
   // RELEASE
-  listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids);
+  listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids,
+                         &peer_ack_ctxs);
   expect_add_event(mock_threads);
-  listener_release_images(mock_listener, shuffled_global_image_ids);
+  listener_release_images(mock_listener, shuffled_global_image_ids,
+                          &peer_ack_ctxs);
 
   mock_image_map->update_images("uuid1", {}, std::move(shuffled_global_image_ids));
   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size() * 2));
@@ -1023,7 +1127,8 @@ TEST_F(TestMockImageMap, RemoveInstanceWithRemoveImage) {
   // instance failed -- update policy for instance removal
   mock_image_map->update_instances_removed({"9876"});
 
-  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, -EBLACKLISTED);
+  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids,
+                         -EBLACKLISTED, &peer_ack_ctxs);
 
   wait_for_scheduled_task();
   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
@@ -1057,7 +1162,9 @@ TEST_F(TestMockImageMap, AddErrorAndRemoveImage) {
   MockUpdateRequest mock_update_request;
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, global_image_ids);
+  std::map<std::string, Context*> peer_ack_ctxs;
+  listener_acquire_images(mock_listener, global_image_ids,
+                          &peer_ack_ctxs);
 
   // initial image list
   mock_image_map->update_images("uuid1", std::move(global_image_ids), {});
@@ -1066,13 +1173,15 @@ TEST_F(TestMockImageMap, AddErrorAndRemoveImage) {
   ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size()));
 
   // remote peer ACKs image acquire request
-  remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   std::set<std::string> shuffled_global_image_ids;
 
   // RELEASE+UPDATE_MAPPING+ACQUIRE
   expect_add_event(mock_threads);
-  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids);
+  expect_listener_images_unmapped(mock_listener, &shuffled_global_image_ids,
+                                  &peer_ack_ctxs);
 
   mock_image_map->update_instances_added({"9876"});
 
@@ -1080,32 +1189,40 @@ TEST_F(TestMockImageMap, AddErrorAndRemoveImage) {
   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size()));
 
   update_map_and_acquire(mock_threads, mock_update_request,
-                         mock_listener, shuffled_global_image_ids, 0);
-  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, 0);
+                         mock_listener, shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
+  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
+                                0, &peer_ack_ctxs);
 
   mock_image_map->update_instances_removed({"9876"});
 
   // instance blacklisted -- ACQUIRE request fails
   update_map_and_acquire(mock_threads, mock_update_request,
-                         mock_listener, shuffled_global_image_ids, 0);
-  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, -EBLACKLISTED);
+                         mock_listener, shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
+  remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids,
+                                -EBLACKLISTED, &peer_ack_ctxs);
 
   // new peer acks acquire request
-  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0);
+  remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0,
+                         &peer_ack_ctxs);
   wait_for_scheduled_task();
 
   std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids);
 
   // remove image
-  listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids);
+  listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids,
+                         &peer_ack_ctxs);
   expect_add_event(mock_threads);
-  listener_release_images(mock_listener, shuffled_global_image_ids);
+  listener_release_images(mock_listener, shuffled_global_image_ids,
+                          &peer_ack_ctxs);
   update_map_request(mock_threads, mock_update_request, shuffled_global_image_ids, 0);
 
   mock_image_map->update_images("uuid1", {}, std::move(shuffled_global_image_ids));
   ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size() * 2));
 
-  remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids_ack, 0);
+  remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids_ack, 0,
+                       &peer_ack_ctxs);
 
   wait_for_scheduled_task();
   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));
@@ -1151,7 +1268,9 @@ TEST_F(TestMockImageMap, MirrorUUIDUpdated) {
   MockUpdateRequest mock_update_request;
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, initial_remote_global_image_ids);
+  std::map<std::string, Context*> peer_ack_ctxs;
+  listener_acquire_images(mock_listener, initial_remote_global_image_ids,
+                          &peer_ack_ctxs);
 
   // initial remote image list
   mock_image_map->update_images("uuid1", std::move(initial_remote_global_image_ids), {});
@@ -1160,24 +1279,31 @@ TEST_F(TestMockImageMap, MirrorUUIDUpdated) {
   ASSERT_TRUE(wait_for_listener_notify(initial_remote_global_image_ids_ack.size()));
 
   // remote peer ACKs image acquire request
-  remote_peer_ack_nowait(mock_image_map.get(), initial_remote_global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(),
+                         initial_remote_global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   // RELEASE+REMOVE_MAPPING
-  listener_remove_images(mock_listener, "uuid1", remote_removed_global_image_ids);
+  listener_remove_images(mock_listener, "uuid1", remote_removed_global_image_ids,
+                         &peer_ack_ctxs);
   expect_add_event(mock_threads);
-  listener_release_images(mock_listener, remote_removed_global_image_ids);
+  listener_release_images(mock_listener, remote_removed_global_image_ids,
+                          &peer_ack_ctxs);
   update_map_request(mock_threads, mock_update_request, remote_removed_global_image_ids, 0);
 
   mock_image_map->update_images("uuid1", {}, std::move(remote_removed_global_image_ids));
   ASSERT_TRUE(wait_for_listener_notify(remote_removed_global_image_ids_ack.size() * 2));
 
-  remote_peer_ack_wait(mock_image_map.get(), remote_removed_global_image_ids_ack, 0);
+  remote_peer_ack_wait(mock_image_map.get(),
+                       remote_removed_global_image_ids_ack, 0,
+                       &peer_ack_ctxs);
 
   // UPDATE_MAPPING+ACQUIRE
   expect_add_event(mock_threads);
   expect_update_request(mock_update_request, 0);
   expect_add_event(mock_threads);
-  listener_acquire_images(mock_listener, remote_added_global_image_ids);
+  listener_acquire_images(mock_listener, remote_added_global_image_ids,
+                          &peer_ack_ctxs);
 
   mock_image_map->update_images("uuid2", std::move(remote_added_global_image_ids), {});
 
@@ -1185,7 +1311,9 @@ TEST_F(TestMockImageMap, MirrorUUIDUpdated) {
   ASSERT_TRUE(wait_for_listener_notify(remote_added_global_image_ids_ack.size()));
 
   // remote peer ACKs image acquire request
-  remote_peer_ack_nowait(mock_image_map.get(), remote_added_global_image_ids_ack, 0);
+  remote_peer_ack_nowait(mock_image_map.get(),
+                         remote_added_global_image_ids_ack, 0,
+                         &peer_ack_ctxs);
 
   wait_for_scheduled_task();
   ASSERT_EQ(0, when_shut_down(mock_image_map.get()));

diff --git a/src/tools/rbd_mirror/ImageMap.cc b/src/tools/rbd_mirror/ImageMap.cc
index 261d02f6c1756f6cc8f6422d41837dd3549b0326..b945e2c9ac8763c56e1a027c17ef7a7d67eff20e 100644 (file)
--- a/src/tools/rbd_mirror/ImageMap.cc
+++ b/src/tools/rbd_mirror/ImageMap.cc
@@ -27,6 +27,22 @@ using image_map::Policy;
 using librbd::util::unique_lock_name;
 using librbd::util::create_async_context_callback;
 
+template <typename I>
+struct ImageMap<I>::C_NotifyInstance : public Context {
+  ImageMap* image_map;
+  std::string global_image_id;
+
+  C_NotifyInstance(ImageMap* image_map, const std::string& global_image_id)
+    : image_map(image_map), global_image_id(global_image_id) {
+    image_map->start_async_op();
+  }
+
+  void finish(int r) override {
+    image_map->handle_peer_ack(global_image_id, r);
+    image_map->finish_async_op();
+  }
+};
+
 template <typename I>
 ImageMap<I>::ImageMap(librados::IoCtx &ioctx, Threads<I> *threads, image_map::Listener &listener)
   : m_ioctx(ioctx),
@@ -251,17 +267,25 @@ void ImageMap<I>::schedule_action(const std::string &global_image_id) {
 }
 
 template <typename I>
-void ImageMap<I>::notify_listener_acquire_release_images(const Updates &acquire,
-                                                         const Updates &release) {
+void ImageMap<I>::notify_listener_acquire_release_images(
+    const Updates &acquire, const Updates &release) {
   dout(20) << ": acquire_count: " << acquire.size() << ", release_count="
            << release.size() << dendl;
 
   for (auto const &update : acquire) {
-    m_listener.acquire_image(update.global_image_id, update.instance_id);
+    m_listener.acquire_image(
+      update.global_image_id, update.instance_id,
+      create_async_context_callback(
+        m_threads->work_queue,
+        new C_NotifyInstance(this, update.global_image_id)));
   }
 
   for (auto const &update : release) {
-    m_listener.release_image(update.global_image_id, update.instance_id);
+    m_listener.release_image(
+      update.global_image_id, update.instance_id,
+      create_async_context_callback(
+        m_threads->work_queue,
+        new C_NotifyInstance(this, update.global_image_id)));
   }
 }
 
@@ -272,7 +296,11 @@ void ImageMap<I>::notify_listener_remove_images(const std::string &peer_uuid,
            << dendl;
 
   for (auto const &update : remove) {
-    m_listener.remove_image(peer_uuid, update.global_image_id, update.instance_id);
+    m_listener.remove_image(
+      peer_uuid, update.global_image_id, update.instance_id,
+      create_async_context_callback(
+        m_threads->work_queue,
+        new C_NotifyInstance(this, update.global_image_id)));
   }
 }
 

diff --git a/src/tools/rbd_mirror/ImageMap.h b/src/tools/rbd_mirror/ImageMap.h
index a10b84060513190eabd757b98c57bf2a016097b0..ffe91742bb31acdbfe648a467e7f7beb6c713d3b 100644 (file)
--- a/src/tools/rbd_mirror/ImageMap.h
+++ b/src/tools/rbd_mirror/ImageMap.h
@@ -43,14 +43,13 @@ public:
                      std::set<std::string> &&added_global_image_ids,
                      std::set<std::string> &&removed_global_image_ids);
 
-  // handle notify response from remote peer (r : 0 == success, negative otherwise)
-  void handle_peer_ack(const std::string &global_image_id, int r);
-
   // add/remove instances
   void update_instances_added(const std::vector<std::string> &instances);
   void update_instances_removed(const std::vector<std::string> &instances);
 
 private:
+  struct C_NotifyInstance;
+
   ImageMap(librados::IoCtx &ioctx, Threads<ImageCtxT> *threads,
            image_map::Listener &listener);
 
@@ -188,6 +187,8 @@ private:
   bool add_peer(const std::string &global_image_id, const std::string &peer_uuid);
   bool remove_peer(const std::string &global_image_id, const std::string &peer_uuid);
 
+  void handle_peer_ack(const std::string &global_image_id, int r);
+
   // queue on-disk,acquire,remove updates in appropriate list
   void queue_update_map(const std::string &global_image_id);
   void queue_remove_map(const std::string &global_image_id);

diff --git a/src/tools/rbd_mirror/image_map/Types.h b/src/tools/rbd_mirror/image_map/Types.h
index 32fe6c24011f02940ab8b0e6f0bfd65223b7e8e5..56433fa1b1cd2c646a5961c015caa58769de8d7d 100644 (file)
--- a/src/tools/rbd_mirror/image_map/Types.h
+++ b/src/tools/rbd_mirror/image_map/Types.h
@@ -13,6 +13,8 @@
 #include "include/utime.h"
 #include "tools/rbd_mirror/types.h"
 
+struct Context;
+
 namespace ceph {
 class Formatter;
 }
@@ -28,12 +30,15 @@ struct Listener {
   }
 
   virtual void acquire_image(const std::string &global_image_id,
-                             const std::string &instance_id) = 0;
+                             const std::string &instance_id,
+                             Context* on_finish) = 0;
   virtual void release_image(const std::string &global_image_id,
-                             const std::string &instance_id) = 0;
+                             const std::string &instance_id,
+                             Context* on_finish) = 0;
   virtual void remove_image(const std::string &mirror_uuid,
                             const std::string &global_image_id,
-                            const std::string &instance_id) = 0;
+                            const std::string &instance_id,
+                            Context* on_finish) = 0;
 };
 
 struct LookupInfo {