rbd-mirror: pool watcher should retrieve the mirror uuid

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/test/rbd_mirror/test_PoolWatcher.cc b/src/test/rbd_mirror/test_PoolWatcher.cc
index 76ba18c565a5d9c2d480317b9c209cc02ab88613..dd857423ee08fe3fad91d6b21de0a35a248b1719 100644 (file)
--- a/src/test/rbd_mirror/test_PoolWatcher.cc
+++ b/src/test/rbd_mirror/test_PoolWatcher.cc
@@ -75,7 +75,8 @@ public:
     PoolWatcherListener(TestPoolWatcher *test) : test(test) {
     }
 
-    void handle_update(const ImageIds &added_image_ids,
+    void handle_update(const std::string &mirror_uuid,
+                       const ImageIds &added_image_ids,
                        const ImageIds &removed_image_ids) override {
       Mutex::Locker locker(test->m_lock);
       for (auto &image_id : removed_image_ids) {

diff --git a/src/test/rbd_mirror/test_mock_PoolWatcher.cc b/src/test/rbd_mirror/test_mock_PoolWatcher.cc
index 334e6c9ae2ac0535ceadade94b96da06d4dd9dc8..ebb26b9f22329ff708acd7cc75cf1b60bcbfc3aa 100644 (file)
--- a/src/test/rbd_mirror/test_mock_PoolWatcher.cc
+++ b/src/test/rbd_mirror/test_mock_PoolWatcher.cc
@@ -163,7 +163,8 @@ public:
     MockListener(TestMockPoolWatcher *test) : test(test) {
     }
 
-    MOCK_METHOD2(handle_update, void(const ImageIds &, const ImageIds &));
+    MOCK_METHOD3(handle_update, void(const std::string &, const ImageIds &,
+                                     const ImageIds &));
   };
 
   TestMockPoolWatcher() : m_lock("TestMockPoolWatcher::m_lock") {
@@ -204,9 +205,11 @@ public:
   }
 
   void expect_listener_handle_update(MockListener &mock_listener,
+                                     const std::string &mirror_uuid,
                                      const ImageIds &added_image_ids,
                                      const ImageIds &removed_image_ids) {
-    EXPECT_CALL(mock_listener, handle_update(added_image_ids, removed_image_ids))
+    EXPECT_CALL(mock_listener, handle_update(mirror_uuid, added_image_ids,
+                                             removed_image_ids))
       .WillOnce(WithoutArgs(Invoke([this]() {
           Mutex::Locker locker(m_lock);
           ++m_update_count;
@@ -214,6 +217,20 @@ public:
         })));
   }
 
+  void expect_mirror_uuid_get(librados::IoCtx &io_ctx,
+                              const std::string &uuid, int r) {
+    bufferlist out_bl;
+    ::encode(uuid, out_bl);
+
+    EXPECT_CALL(get_mock_io_ctx(io_ctx),
+                exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"),
+                     _, _, _))
+      .WillOnce(DoAll(WithArg<5>(Invoke([this, out_bl](bufferlist *bl) {
+                          *bl = out_bl;
+                        })),
+                      Return(r)));
+  }
+
   void expect_dir_list(librados::IoCtx &io_ctx,
                        const std::string &id, const std::string &name, int r) {
     bufferlist in_bl;
@@ -298,9 +315,10 @@ TEST_F(TestMockPoolWatcher, EmptyPool) {
 
   MockRefreshImagesRequest mock_refresh_images_request;
   expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, {}, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 
   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
                                     mock_listener);
@@ -327,9 +345,10 @@ TEST_F(TestMockPoolWatcher, NonEmptyPool) {
     {"global id 2", "remote id 2", "image name 2"}};
   MockRefreshImagesRequest mock_refresh_images_request;
   expect_refresh_images(mock_refresh_images_request, image_ids, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, image_ids, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {});
 
   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
                                     mock_listener);
@@ -370,11 +389,13 @@ TEST_F(TestMockPoolWatcher, NotifyDuringRefresh) {
   expect_dir_list(m_remote_io_ctx, "remote id 3", "image name 3", 0);
   expect_dir_list(m_remote_io_ctx, "dummy", "", -ENOENT);
 
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
+
   MockListener mock_listener(this);
   image_ids = {
     {"global id 1", "remote id 1a", "image name 1a"},
     {"global id 3", "remote id 3", "image name 3"}};
-  expect_listener_handle_update(mock_listener, image_ids, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {});
 
   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
                                     mock_listener);
@@ -417,13 +438,15 @@ TEST_F(TestMockPoolWatcher, Notify) {
     {"global id 2", "remote id 2", "image name 2"}};
   MockRefreshImagesRequest mock_refresh_images_request;
   expect_refresh_images(mock_refresh_images_request, image_ids, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
+
   EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
     .WillOnce(Invoke([this](Context *ctx, int r) {
         m_threads->work_queue->queue(ctx, r);
       }));
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, image_ids, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {});
 
   Context *notify_ctx = nullptr;
   EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
@@ -437,7 +460,7 @@ TEST_F(TestMockPoolWatcher, Notify) {
   expect_dir_list(m_remote_io_ctx, "remote id 3", "image name 3", 0);
   expect_dir_list(m_remote_io_ctx, "dummy", "", -ENOENT);
   expect_listener_handle_update(
-    mock_listener,
+    mock_listener, "remote uuid",
     {{"global id 1", "remote id 1a", "image name 1a"},
      {"global id 3", "remote id 3", "image name 3"}},
     {{"global id 1", "remote id 1", "image name 1"},
@@ -509,9 +532,10 @@ TEST_F(TestMockPoolWatcher, RegisterWatcherMissing) {
 
   MockRefreshImagesRequest mock_refresh_images_request;
   expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, {}, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 
   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
                                     mock_listener);
@@ -539,9 +563,10 @@ TEST_F(TestMockPoolWatcher, RegisterWatcherError) {
 
   MockRefreshImagesRequest mock_refresh_images_request;
   expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, {}, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 
   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
                                     mock_listener);
@@ -589,9 +614,10 @@ TEST_F(TestMockPoolWatcher, RefreshMissing) {
 
   MockRefreshImagesRequest mock_refresh_images_request;
   expect_refresh_images(mock_refresh_images_request, {}, -ENOENT);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, {}, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 
   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
                                     mock_listener);
@@ -619,9 +645,99 @@ TEST_F(TestMockPoolWatcher, RefreshError) {
 
   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
   expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
+
+  MockListener mock_listener(this);
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
+
+  MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
+                                    mock_listener);
+  C_SaferCond ctx;
+  mock_pool_watcher.init(&ctx);
+  ASSERT_EQ(0, ctx.wait());
+
+  ASSERT_TRUE(wait_for_update(1));
+  expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
+  ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
+}
+
+TEST_F(TestMockPoolWatcher, GetMirrorUuidBlacklist) {
+  MockThreads mock_threads(m_threads);
+  expect_work_queue(mock_threads);
+
+  InSequence seq;
+  MockMirroringWatcher mock_mirroring_watcher;
+  expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
+  expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
+
+  MockRefreshImagesRequest mock_refresh_images_request;
+  expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", -EBLACKLISTED);
+
+  MockListener mock_listener(this);
+  MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
+                                    mock_listener);
+  C_SaferCond ctx;
+  mock_pool_watcher.init(&ctx);
+  ASSERT_EQ(-EBLACKLISTED, ctx.wait());
+  ASSERT_TRUE(mock_pool_watcher.is_blacklisted());
+
+  expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
+  ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
+}
+
+TEST_F(TestMockPoolWatcher, GetMirrorUuidMissing) {
+  MockThreads mock_threads(m_threads);
+  expect_work_queue(mock_threads);
+
+  InSequence seq;
+  MockMirroringWatcher mock_mirroring_watcher;
+  expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
+  expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
+
+  MockRefreshImagesRequest mock_refresh_images_request;
+  expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "", -ENOENT);
+  expect_timer_add_event(mock_threads);
+
+  expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
+  expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, {}, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
+
+  MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
+                                    mock_listener);
+  C_SaferCond ctx;
+  mock_pool_watcher.init(&ctx);
+  ASSERT_EQ(-ENOENT, ctx.wait());
+
+  ASSERT_TRUE(wait_for_update(1));
+  expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
+  ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
+}
+
+TEST_F(TestMockPoolWatcher, GetMirrorUuidError) {
+  MockThreads mock_threads(m_threads);
+  expect_work_queue(mock_threads);
+
+  InSequence seq;
+  MockMirroringWatcher mock_mirroring_watcher;
+  expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
+  expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
+
+  MockRefreshImagesRequest mock_refresh_images_request;
+  expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", -EINVAL);
+  expect_timer_add_event(mock_threads);
+
+  expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
+  expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
+
+  MockListener mock_listener(this);
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 
   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
                                     mock_listener);
@@ -645,14 +761,17 @@ TEST_F(TestMockPoolWatcher, Rewatch) {
 
   MockRefreshImagesRequest mock_refresh_images_request;
   expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, {}, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 
   expect_timer_add_event(mock_threads);
   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
   expect_refresh_images(mock_refresh_images_request, {{"global id", "image id", "name"}}, 0);
-  expect_listener_handle_update(mock_listener, {{"global id", "image id", "name"}}, {});
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
+  expect_listener_handle_update(mock_listener, "remote uuid",
+                                {{"global id", "image id", "name"}}, {});
 
   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
                                     mock_listener);
@@ -679,9 +798,10 @@ TEST_F(TestMockPoolWatcher, RewatchBlacklist) {
 
   MockRefreshImagesRequest mock_refresh_images_request;
   expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, {}, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 
   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
                                     mock_listener);
@@ -708,14 +828,17 @@ TEST_F(TestMockPoolWatcher, RewatchError) {
 
   MockRefreshImagesRequest mock_refresh_images_request;
   expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, {}, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 
   expect_timer_add_event(mock_threads);
   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
   expect_refresh_images(mock_refresh_images_request, {{"global id", "image id", "name"}}, 0);
-  expect_listener_handle_update(mock_listener, {{"global id", "image id", "name"}}, {});
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
+  expect_listener_handle_update(mock_listener, "remote uuid",
+                                {{"global id", "image id", "name"}}, {});
 
   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
                                     mock_listener);
@@ -742,9 +865,10 @@ TEST_F(TestMockPoolWatcher, GetImageNameBlacklist) {
 
   MockRefreshImagesRequest mock_refresh_images_request;
   expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, {}, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 
   expect_dir_list(m_remote_io_ctx, "remote id", "image name", -EBLACKLISTED);
 
@@ -780,16 +904,19 @@ TEST_F(TestMockPoolWatcher, GetImageNameError) {
 
   MockRefreshImagesRequest mock_refresh_images_request;
   expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, {}, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 
   expect_dir_list(m_remote_io_ctx, "remote id", "image name", -EINVAL);
   expect_timer_add_event(mock_threads);
 
   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
   expect_refresh_images(mock_refresh_images_request, {{"global id", "remote id", "name"}}, 0);
-  expect_listener_handle_update(mock_listener, {{"global id", "remote id", "name"}}, {});
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
+  expect_listener_handle_update(mock_listener, "remote uuid",
+                                {{"global id", "remote id", "name"}}, {});
 
   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
                                     mock_listener);
@@ -824,23 +951,69 @@ TEST_F(TestMockPoolWatcher, DeferredRefresh) {
         MirroringWatcher::get_instance().handle_rewatch_complete(0);
         mock_refresh_images_request.on_finish->complete(0);
         }));
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
   expect_timer_add_event(mock_threads);
+
   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
   expect_refresh_images(mock_refresh_images_request, {}, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 
   MockListener mock_listener(this);
-  expect_listener_handle_update(mock_listener, {}, {});
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 
   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
                                     mock_listener);
   C_SaferCond ctx;
   mock_pool_watcher.init(&ctx);
   ASSERT_EQ(0, ctx.wait());
-
   ASSERT_TRUE(wait_for_update(1));
+
   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 }
 
+TEST_F(TestMockPoolWatcher, MirrorUuidUpdated) {
+  MockThreads mock_threads(m_threads);
+  expect_work_queue(mock_threads);
+
+  InSequence seq;
+  MockMirroringWatcher mock_mirroring_watcher;
+  expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
+  expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
+
+  ImageIds image_ids{
+    {"global id 1", "remote id 1", "image name 1"},
+    {"global id 2", "remote id 2", "image name 2"}};
+  MockRefreshImagesRequest mock_refresh_images_request;
+  expect_refresh_images(mock_refresh_images_request, image_ids, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
+
+  MockListener mock_listener(this);
+  expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {});
+
+  MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
+                                    mock_listener);
+  C_SaferCond ctx;
+  mock_pool_watcher.init(&ctx);
+  ASSERT_EQ(0, ctx.wait());
+
+  ASSERT_TRUE(wait_for_update(1));
+
+  expect_timer_add_event(mock_threads);
+  ImageIds new_image_ids{
+    {"global id 1", "remote id 1", "image name 1"}};
+  expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
+  expect_refresh_images(mock_refresh_images_request, new_image_ids, 0);
+  expect_mirror_uuid_get(m_remote_io_ctx, "updated uuid", 0);
+  expect_listener_handle_update(mock_listener, "remote uuid", {}, image_ids);
+  expect_listener_handle_update(mock_listener, "updated uuid", new_image_ids,
+                                {});
+
+  MirroringWatcher::get_instance().handle_rewatch_complete(0);
+  ASSERT_TRUE(wait_for_update(2));
+
+  expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
+  ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
+}
 } // namespace mirror
 } // namespace rbd

diff --git a/src/tools/rbd_mirror/PoolWatcher.cc b/src/tools/rbd_mirror/PoolWatcher.cc
index aa2016f4cbe31e792f29908039c58d4def4f3b8a..7bdd8cdb09ec283e9c5537bb8ec98d08cbba9bdd 100644 (file)
--- a/src/tools/rbd_mirror/PoolWatcher.cc
+++ b/src/tools/rbd_mirror/PoolWatcher.cc
@@ -51,7 +51,8 @@ public:
   }
 
   void handle_mode_updated(cls::rbd::MirrorMode mirror_mode) override {
-    // do nothing
+    // invalidate all image state and refresh the pool contents
+    m_pool_watcher->schedule_refresh_images(5);
   }
 
   void handle_image_updated(cls::rbd::MirrorImageState state,
@@ -244,6 +245,66 @@ template <typename I>
 void PoolWatcher<I>::handle_refresh_images(int r) {
   dout(5) << "r=" << r << dendl;
 
+  bool retry_refresh = false;
+  Context *on_init_finish = nullptr;
+  {
+    Mutex::Locker locker(m_lock);
+    assert(m_image_ids_invalid);
+    assert(m_refresh_in_progress);
+
+    if (r >= 0) {
+      m_pending_image_ids = std::move(m_refresh_image_ids);
+    } else if (r == -EBLACKLISTED) {
+      dout(0) << "detected client is blacklisted during image refresh" << dendl;
+
+      m_blacklisted = true;
+      m_refresh_in_progress = false;
+      std::swap(on_init_finish, m_on_init_finish);
+    } else if (r == -ENOENT) {
+      dout(5) << "mirroring directory not found" << dendl;
+      m_pending_image_ids.clear();
+      r = 0;
+    } else {
+      m_refresh_in_progress = false;
+      retry_refresh = true;
+    }
+  }
+
+  if (retry_refresh) {
+    derr << "failed to retrieve mirroring directory: " << cpp_strerror(r)
+         << dendl;
+    schedule_refresh_images(10);
+  } else if (r >= 0) {
+    get_mirror_uuid();
+    return;
+  }
+
+  m_async_op_tracker.finish_op();
+  if (on_init_finish != nullptr) {
+    assert(r == -EBLACKLISTED);
+    on_init_finish->complete(r);
+  }
+}
+
+template <typename I>
+void PoolWatcher<I>::get_mirror_uuid() {
+  dout(5) << dendl;
+
+  librados::ObjectReadOperation op;
+  librbd::cls_client::mirror_uuid_get_start(&op);
+
+  m_out_bl.clear();
+  librados::AioCompletion *aio_comp = create_rados_callback<
+    PoolWatcher, &PoolWatcher<I>::handle_get_mirror_uuid>(this);
+  int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl);
+  assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void PoolWatcher<I>::handle_get_mirror_uuid(int r) {
+  dout(5) << "r=" << r << dendl;
+
   bool deferred_refresh = false;
   bool retry_refresh = false;
   Context *on_init_finish = nullptr;
@@ -253,12 +314,22 @@ void PoolWatcher<I>::handle_refresh_images(int r) {
     assert(m_refresh_in_progress);
     m_refresh_in_progress = false;
 
+    m_pending_mirror_uuid = "";
+    if (r >= 0) {
+      bufferlist::iterator it = m_out_bl.begin();
+      r = librbd::cls_client::mirror_uuid_get_finish(
+        &it, &m_pending_mirror_uuid);
+    }
+    if (r >= 0 && m_pending_mirror_uuid.empty()) {
+      r = -ENOENT;
+    }
+
     if (m_deferred_refresh) {
       // need to refresh -- skip the notification
       deferred_refresh = true;
     } else if (r >= 0) {
+      dout(10) << "mirror_uuid=" << m_pending_mirror_uuid << dendl;
       m_image_ids_invalid = false;
-      m_pending_image_ids = m_refresh_image_ids;
       std::swap(on_init_finish, m_on_init_finish);
       schedule_listener();
     } else if (r == -EBLACKLISTED) {
@@ -267,12 +338,9 @@ void PoolWatcher<I>::handle_refresh_images(int r) {
       m_blacklisted = true;
       std::swap(on_init_finish, m_on_init_finish);
     } else if (r == -ENOENT) {
-      dout(5) << "mirroring directory not found" << dendl;
-      m_image_ids_invalid = false;
-      m_pending_image_ids.clear();
+      dout(5) << "mirroring uuid not found" << dendl;
       std::swap(on_init_finish, m_on_init_finish);
-      r = 0;
-      schedule_listener();
+      retry_refresh = true;
     } else {
       retry_refresh = true;
     }
@@ -282,7 +350,7 @@ void PoolWatcher<I>::handle_refresh_images(int r) {
     dout(5) << "scheduling deferred refresh" << dendl;
     schedule_refresh_images(0);
   } else if (retry_refresh) {
-    derr << "failed to retrieve mirroring directory: " << cpp_strerror(r)
+    derr << "failed to retrieve mirror uuid: " << cpp_strerror(r)
          << dendl;
     schedule_refresh_images(10);
   }
@@ -495,8 +563,33 @@ template <typename I>
 void PoolWatcher<I>::notify_listener() {
   dout(10) << dendl;
 
+  std::string mirror_uuid;
   ImageIds added_image_ids;
   ImageIds removed_image_ids;
+  {
+    Mutex::Locker locker(m_lock);
+    assert(m_notify_listener_in_progress);
+
+    // if the mirror uuid is updated, treat it as the removal of all
+    // images in the pool
+    if (m_mirror_uuid != m_pending_mirror_uuid) {
+      if (!m_mirror_uuid.empty()) {
+        dout(0) << "mirror uuid updated:"
+                << "old=" << m_mirror_uuid << ", "
+                << "new=" << m_pending_mirror_uuid << dendl;
+      }
+
+      mirror_uuid = m_mirror_uuid;
+      removed_image_ids = std::move(m_image_ids);
+      m_image_ids.clear();
+    }
+  }
+
+  if (!removed_image_ids.empty()) {
+    m_listener.handle_update(mirror_uuid, {}, removed_image_ids);
+    removed_image_ids.clear();
+  }
+
   {
     Mutex::Locker locker(m_lock);
     assert(m_notify_listener_in_progress);
@@ -553,9 +646,12 @@ void PoolWatcher<I>::notify_listener() {
 
     m_pending_updates = false;
     m_image_ids = m_pending_image_ids;
+
+    m_mirror_uuid = m_pending_mirror_uuid;
+    mirror_uuid = m_mirror_uuid;
   }
 
-  m_listener.handle_update(added_image_ids, removed_image_ids);
+  m_listener.handle_update(mirror_uuid, added_image_ids, removed_image_ids);
 
   {
     Mutex::Locker locker(m_lock);

diff --git a/src/tools/rbd_mirror/PoolWatcher.h b/src/tools/rbd_mirror/PoolWatcher.h
index a8aa32948bb25b0cac3a98bd8b088bfa92b1d5f7..747a0ee5dc882f0180635a0003174abcf625e4fa 100644 (file)
--- a/src/tools/rbd_mirror/PoolWatcher.h
+++ b/src/tools/rbd_mirror/PoolWatcher.h
@@ -38,7 +38,8 @@ public:
     virtual ~Listener() {
     }
 
-    virtual void handle_update(const ImageIds &added_image_ids,
+    virtual void handle_update(const std::string &mirror_uuid,
+                               const ImageIds &added_image_ids,
                                const ImageIds &removed_image_ids) = 0;
   };
 
@@ -73,6 +74,9 @@ private:
    *    |/----------------------------\   |
    *    |                             |   |
    *    v                             |   |
+   * GET_MIRROR_UUID                  |   |
+   *    |                             |   |
+   *    v                             |   |
    * NOTIFY_LISTENER                  |   |
    *    |                             |   |
    *    v                             |   |
@@ -134,6 +138,7 @@ private:
   Context *m_on_init_finish = nullptr;
 
   ImageIds m_image_ids;
+  std::string m_mirror_uuid;
 
   bool m_pending_updates = false;
   bool m_notify_listener_in_progress = false;
@@ -141,6 +146,8 @@ private:
   ImageIds m_pending_added_image_ids;
   ImageIds m_pending_removed_image_ids;
 
+  std::string m_pending_mirror_uuid;
+
   MirroringWatcher *m_mirroring_watcher;
 
   Context *m_timer_ctx = nullptr;
@@ -166,6 +173,9 @@ private:
   void schedule_refresh_images(double interval);
   void process_refresh_images();
 
+  void get_mirror_uuid();
+  void handle_get_mirror_uuid(int r);
+
   void handle_rewatch_complete(int r);
   void handle_image_updated(const std::string &remote_image_id,
                             const std::string &global_image_id,

diff --git a/src/tools/rbd_mirror/Replayer.cc b/src/tools/rbd_mirror/Replayer.cc
index f8ee473e19292c59fe16c5accd9126033111bc12..953db35f63ed530060fedc9adcd6dfdec418c01d 100644 (file)
--- a/src/tools/rbd_mirror/Replayer.cc
+++ b/src/tools/rbd_mirror/Replayer.cc
@@ -568,8 +568,10 @@ void Replayer::release_leader()
   m_leader_watcher->release_leader();
 }
 
-void Replayer::handle_update(const ImageIds &added_image_ids,
+void Replayer::handle_update(const std::string &mirror_uuid,
+                             const ImageIds &added_image_ids,
                              const ImageIds &removed_image_ids) {
+  assert(!mirror_uuid.empty());
   if (m_stopping.read()) {
     return;
   }
@@ -605,9 +607,7 @@ void Replayer::handle_update(const ImageIds &added_image_ids,
   for (auto &image_id : removed_image_ids) {
     auto image_it = m_image_replayers.find(image_id.global_id);
     if (image_it != m_image_replayers.end()) {
-      assert(!m_remote_mirror_uuid.empty());
-      image_it->second->remove_remote_image(m_remote_mirror_uuid,
-                                            image_id.id);
+      image_it->second->remove_remote_image(mirror_uuid, image_id.id);
 
       if (image_it->second->is_running()) {
         dout(20) << "stop image replayer for remote image "
@@ -647,16 +647,6 @@ void Replayer::handle_update(const ImageIds &added_image_ids,
     return;
   }
 
-  std::string remote_mirror_uuid;
-  r = librbd::cls_client::mirror_uuid_get(&m_remote_io_ctx,
-                                          &remote_mirror_uuid);
-  if (r < 0 || remote_mirror_uuid.empty()) {
-    derr << "failed to retrieve remote mirror uuid from pool "
-         << m_remote_io_ctx.get_pool_name() << ": " << cpp_strerror(r) << dendl;
-    return;
-  }
-  m_remote_mirror_uuid = remote_mirror_uuid;
-
   // start replayers for newly added remote image sources
   for (auto &image_id : added_image_ids) {
     auto it = m_image_replayers.find(image_id.global_id);
@@ -672,7 +662,7 @@ void Replayer::handle_update(const ImageIds &added_image_ids,
         std::make_pair(image_id.global_id, std::move(image_replayer))).first;
     }
 
-    it->second->add_remote_image(remote_mirror_uuid, image_id.id,
+    it->second->add_remote_image(mirror_uuid, image_id.id,
                                  m_remote_io_ctx);
     if (!it->second->is_running()) {
       dout(20) << "starting image replayer for remote image "

diff --git a/src/tools/rbd_mirror/Replayer.h b/src/tools/rbd_mirror/Replayer.h
index f503d1495fe9dddd2fc8543f756b671829cd6fbb..6eb3753aa3b963015588fa0dc33bd0622eef734b 100644 (file)
--- a/src/tools/rbd_mirror/Replayer.h
+++ b/src/tools/rbd_mirror/Replayer.h
@@ -66,15 +66,17 @@ private:
     PoolWatcherListener(Replayer *replayer) : replayer(replayer) {
     }
 
-    void handle_update(const ImageIds &added_image_ids,
+    void handle_update(const std::string &mirror_uuid,
+                       const ImageIds &added_image_ids,
                        const ImageIds &removed_image_ids) override {
-      replayer->handle_update(added_image_ids, removed_image_ids);
+      replayer->handle_update(mirror_uuid, added_image_ids, removed_image_ids);
     }
   };
 
   struct C_RefreshLocalImages;
 
-  void handle_update(const ImageIds &added_image_ids,
+  void handle_update(const std::string &mirror_uuid,
+                     const ImageIds &added_image_ids,
                      const ImageIds &removed_image_ids);
 
   void start_image_replayer(unique_ptr<ImageReplayer<> > &image_replayer);
@@ -117,8 +119,6 @@ private:
   int64_t m_local_pool_id = -1;
   int64_t m_remote_pool_id = -1;
 
-  std::string m_remote_mirror_uuid;
-
   PoolWatcherListener m_pool_watcher_listener;
   std::unique_ptr<PoolWatcher<> > m_pool_watcher;