rbd-mirror: re-refresh pool if scheduled while refresh in-flight

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/test/rbd_mirror/test_mock_PoolWatcher.cc b/src/test/rbd_mirror/test_mock_PoolWatcher.cc
index 737aad9061f0e2b7ab420bba8805e3290523459e..334e6c9ae2ac0535ceadade94b96da06d4dd9dc8 100644 (file)
--- a/src/test/rbd_mirror/test_mock_PoolWatcher.cc
+++ b/src/test/rbd_mirror/test_mock_PoolWatcher.cc
@@ -236,8 +236,12 @@ public:
 
   void expect_timer_add_event(MockThreads &mock_threads) {
     EXPECT_CALL(*mock_threads.timer, add_event_after(_, _))
-      .WillOnce(WithArg<1>(Invoke([](Context *ctx) {
-          ctx->complete(0);
+      .WillOnce(WithArg<1>(Invoke([this](Context *ctx) {
+          auto wrapped_ctx = new FunctionContext([this, ctx](int r) {
+              Mutex::Locker timer_locker(m_threads->timer_lock);
+              ctx->complete(r);
+            });
+          m_threads->work_queue->queue(wrapped_ctx, 0);
         })));
   }
 
@@ -803,5 +807,40 @@ TEST_F(TestMockPoolWatcher, GetImageNameError) {
   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 }
 
+TEST_F(TestMockPoolWatcher, DeferredRefresh) {
+  MockThreads mock_threads(m_threads);
+  expect_work_queue(mock_threads);
+
+  InSequence seq;
+  MockMirroringWatcher mock_mirroring_watcher;
+  expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
+  expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
+
+  MockRefreshImagesRequest mock_refresh_images_request;
+
+  EXPECT_CALL(mock_refresh_images_request, send())
+    .WillOnce(Invoke([&mock_refresh_images_request]() {
+        *mock_refresh_images_request.image_ids = {};
+        MirroringWatcher::get_instance().handle_rewatch_complete(0);
+        mock_refresh_images_request.on_finish->complete(0);
+        }));
+  expect_timer_add_event(mock_threads);
+  expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
+  expect_refresh_images(mock_refresh_images_request, {}, 0);
+
+  MockListener mock_listener(this);
+  expect_listener_handle_update(mock_listener, {}, {});
+
+  MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
+                                    mock_listener);
+  C_SaferCond ctx;
+  mock_pool_watcher.init(&ctx);
+  ASSERT_EQ(0, ctx.wait());
+
+  ASSERT_TRUE(wait_for_update(1));
+  expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
+  ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
+}
+
 } // namespace mirror
 } // namespace rbd

diff --git a/src/tools/rbd_mirror/PoolWatcher.cc b/src/tools/rbd_mirror/PoolWatcher.cc
index 270180738e0fe5be46ce9effe1a4722e12a0445b..aa2016f4cbe31e792f29908039c58d4def4f3b8a 100644 (file)
--- a/src/tools/rbd_mirror/PoolWatcher.cc
+++ b/src/tools/rbd_mirror/PoolWatcher.cc
@@ -93,6 +93,9 @@ void PoolWatcher<I>::init(Context *on_finish) {
   {
     Mutex::Locker locker(m_lock);
     m_on_init_finish = on_finish;
+
+    assert(!m_refresh_in_progress);
+    m_refresh_in_progress = true;
   }
 
   // start async updates for mirror image directory
@@ -126,8 +129,7 @@ void PoolWatcher<I>::register_watcher() {
   {
     Mutex::Locker locker(m_lock);
     assert(m_image_ids_invalid);
-    assert(!m_refresh_in_progress);
-    m_refresh_in_progress = true;
+    assert(m_refresh_in_progress);
   }
 
   // if the watch registration is in-flight, let the watcher
@@ -242,6 +244,7 @@ template <typename I>
 void PoolWatcher<I>::handle_refresh_images(int r) {
   dout(5) << "r=" << r << dendl;
 
+  bool deferred_refresh = false;
   bool retry_refresh = false;
   Context *on_init_finish = nullptr;
   {
@@ -250,7 +253,10 @@ void PoolWatcher<I>::handle_refresh_images(int r) {
     assert(m_refresh_in_progress);
     m_refresh_in_progress = false;
 
-    if (r >= 0) {
+    if (m_deferred_refresh) {
+      // need to refresh -- skip the notification
+      deferred_refresh = true;
+    } else if (r >= 0) {
       m_image_ids_invalid = false;
       m_pending_image_ids = m_refresh_image_ids;
       std::swap(on_init_finish, m_on_init_finish);
@@ -272,7 +278,10 @@ void PoolWatcher<I>::handle_refresh_images(int r) {
     }
   }
 
-  if (retry_refresh) {
+  if (deferred_refresh) {
+    dout(5) << "scheduling deferred refresh" << dendl;
+    schedule_refresh_images(0);
+  } else if (retry_refresh) {
     derr << "failed to retrieve mirroring directory: " << cpp_strerror(r)
          << dendl;
     schedule_refresh_images(10);
@@ -289,12 +298,16 @@ void PoolWatcher<I>::schedule_refresh_images(double interval) {
   Mutex::Locker timer_locker(m_threads->timer_lock);
   Mutex::Locker locker(m_lock);
   if (m_shutting_down || m_refresh_in_progress || m_timer_ctx != nullptr) {
+    if (m_refresh_in_progress && !m_deferred_refresh) {
+      dout(5) << "deferring refresh until in-flight refresh completes" << dendl;
+      m_deferred_refresh = true;
+    }
     return;
   }
 
   m_image_ids_invalid = true;
   m_timer_ctx = new FunctionContext([this](int r) {
-      processs_refresh_images();
+      process_refresh_images();
     });
   m_threads->timer->add_event_after(interval, m_timer_ctx);
 }
@@ -437,11 +450,18 @@ void PoolWatcher<I>::handle_get_image_name(int r) {
 }
 
 template <typename I>
-void PoolWatcher<I>::processs_refresh_images() {
+void PoolWatcher<I>::process_refresh_images() {
   assert(m_threads->timer_lock.is_locked());
   assert(m_timer_ctx != nullptr);
   m_timer_ctx = nullptr;
 
+  {
+    Mutex::Locker locker(m_lock);
+    assert(!m_refresh_in_progress);
+    m_refresh_in_progress = true;
+    m_deferred_refresh = false;
+  }
+
   // execute outside of the timer's lock
   m_async_op_tracker.start_op();
   Context *ctx = new FunctionContext([this](int r) {

diff --git a/src/tools/rbd_mirror/PoolWatcher.h b/src/tools/rbd_mirror/PoolWatcher.h
index 1cd2ede2d12aafe4848f2ba771a7428575f72da1..a8aa32948bb25b0cac3a98bd8b088bfa92b1d5f7 100644 (file)
--- a/src/tools/rbd_mirror/PoolWatcher.h
+++ b/src/tools/rbd_mirror/PoolWatcher.h
@@ -150,6 +150,7 @@ private:
   bool m_shutting_down = false;
   bool m_image_ids_invalid = true;
   bool m_refresh_in_progress = false;
+  bool m_deferred_refresh = false;
 
   UpdatedImages m_updated_images;
   IdToUpdatedImages m_id_to_updated_images;
@@ -163,7 +164,7 @@ private:
   void handle_refresh_images(int r);
 
   void schedule_refresh_images(double interval);
-  void processs_refresh_images();
+  void process_refresh_images();
 
   void handle_rewatch_complete(int r);
   void handle_image_updated(const std::string &remote_image_id,