From: Jason Dillaman Date: Wed, 29 Mar 2017 19:09:25 +0000 (-0400) Subject: rbd-mirror: re-refresh pool if scheduled while refresh in-flight X-Git-Tag: v12.0.2~174^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=44b9a53656b7c4c1014b777da8099b463f7a43fa;p=ceph.git rbd-mirror: re-refresh pool if scheduled while refresh in-flight Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/test_mock_PoolWatcher.cc b/src/test/rbd_mirror/test_mock_PoolWatcher.cc index 737aad9061f0..334e6c9ae2ac 100644 --- a/src/test/rbd_mirror/test_mock_PoolWatcher.cc +++ b/src/test/rbd_mirror/test_mock_PoolWatcher.cc @@ -236,8 +236,12 @@ public: void expect_timer_add_event(MockThreads &mock_threads) { EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) - .WillOnce(WithArg<1>(Invoke([](Context *ctx) { - ctx->complete(0); + .WillOnce(WithArg<1>(Invoke([this](Context *ctx) { + auto wrapped_ctx = new FunctionContext([this, ctx](int r) { + Mutex::Locker timer_locker(m_threads->timer_lock); + ctx->complete(r); + }); + m_threads->work_queue->queue(wrapped_ctx, 0); }))); } @@ -803,5 +807,40 @@ TEST_F(TestMockPoolWatcher, GetImageNameError) { ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); } +TEST_F(TestMockPoolWatcher, DeferredRefresh) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + + EXPECT_CALL(mock_refresh_images_request, send()) + .WillOnce(Invoke([&mock_refresh_images_request]() { + *mock_refresh_images_request.image_ids = {}; + MirroringWatcher::get_instance().handle_rewatch_complete(0); + mock_refresh_images_request.on_finish->complete(0); + })); + expect_timer_add_event(mock_threads); + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {}, 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + } // namespace mirror } // namespace rbd diff --git a/src/tools/rbd_mirror/PoolWatcher.cc b/src/tools/rbd_mirror/PoolWatcher.cc index 270180738e0f..aa2016f4cbe3 100644 --- a/src/tools/rbd_mirror/PoolWatcher.cc +++ b/src/tools/rbd_mirror/PoolWatcher.cc @@ -93,6 +93,9 @@ void PoolWatcher::init(Context *on_finish) { { Mutex::Locker locker(m_lock); m_on_init_finish = on_finish; + + assert(!m_refresh_in_progress); + m_refresh_in_progress = true; } // start async updates for mirror image directory @@ -126,8 +129,7 @@ void PoolWatcher::register_watcher() { { Mutex::Locker locker(m_lock); assert(m_image_ids_invalid); - assert(!m_refresh_in_progress); - m_refresh_in_progress = true; + assert(m_refresh_in_progress); } // if the watch registration is in-flight, let the watcher @@ -242,6 +244,7 @@ template void PoolWatcher::handle_refresh_images(int r) { dout(5) << "r=" << r << dendl; + bool deferred_refresh = false; bool retry_refresh = false; Context *on_init_finish = nullptr; { @@ -250,7 +253,10 @@ void PoolWatcher::handle_refresh_images(int r) { assert(m_refresh_in_progress); m_refresh_in_progress = false; - if (r >= 0) { + if (m_deferred_refresh) { + // need to refresh -- skip the notification + deferred_refresh = true; + } else if (r >= 0) { m_image_ids_invalid = false; m_pending_image_ids = m_refresh_image_ids; std::swap(on_init_finish, m_on_init_finish); @@ -272,7 +278,10 @@ void PoolWatcher::handle_refresh_images(int r) { } } - if (retry_refresh) { + if (deferred_refresh) { + dout(5) << "scheduling deferred refresh" << dendl; + schedule_refresh_images(0); + } else if (retry_refresh) { derr << "failed to retrieve mirroring directory: " << cpp_strerror(r) << dendl; schedule_refresh_images(10); @@ -289,12 +298,16 @@ void PoolWatcher::schedule_refresh_images(double interval) { Mutex::Locker timer_locker(m_threads->timer_lock); Mutex::Locker locker(m_lock); if (m_shutting_down || m_refresh_in_progress || m_timer_ctx != nullptr) { + if (m_refresh_in_progress && !m_deferred_refresh) { + dout(5) << "deferring refresh until in-flight refresh completes" << dendl; + m_deferred_refresh = true; + } return; } m_image_ids_invalid = true; m_timer_ctx = new FunctionContext([this](int r) { - processs_refresh_images(); + process_refresh_images(); }); m_threads->timer->add_event_after(interval, m_timer_ctx); } @@ -437,11 +450,18 @@ void PoolWatcher::handle_get_image_name(int r) { } template -void PoolWatcher::processs_refresh_images() { +void PoolWatcher::process_refresh_images() { assert(m_threads->timer_lock.is_locked()); assert(m_timer_ctx != nullptr); m_timer_ctx = nullptr; + { + Mutex::Locker locker(m_lock); + assert(!m_refresh_in_progress); + m_refresh_in_progress = true; + m_deferred_refresh = false; + } + // execute outside of the timer's lock m_async_op_tracker.start_op(); Context *ctx = new FunctionContext([this](int r) { diff --git a/src/tools/rbd_mirror/PoolWatcher.h b/src/tools/rbd_mirror/PoolWatcher.h index 1cd2ede2d12a..a8aa32948bb2 100644 --- a/src/tools/rbd_mirror/PoolWatcher.h +++ b/src/tools/rbd_mirror/PoolWatcher.h @@ -150,6 +150,7 @@ private: bool m_shutting_down = false; bool m_image_ids_invalid = true; bool m_refresh_in_progress = false; + bool m_deferred_refresh = false; UpdatedImages m_updated_images; IdToUpdatedImages m_id_to_updated_images; @@ -163,7 +164,7 @@ private: void handle_refresh_images(int r); void schedule_refresh_images(double interval); - void processs_refresh_images(); + void process_refresh_images(); void handle_rewatch_complete(int r); void handle_image_updated(const std::string &remote_image_id,