From: Jason Dillaman Date: Wed, 25 Sep 2019 16:58:35 +0000 (-0400) Subject: rbd-mirror: mirror status updater optionally pings peer X-Git-Tag: v15.1.0~1245^2~6 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5251f593f5a6aa57088d68a38d81c3f1835baac9;p=ceph.git rbd-mirror: mirror status updater optionally pings peer This will register the local site as a peer in the remote for TX-only scenarios. Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index aa2abfd64ef..8bf1724f6da 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -132,7 +132,7 @@ public: EXPECT_EQ(0, m_local_ioctx.create(RBD_MIRRORING, false)); m_local_status_updater = rbd::mirror::MirrorStatusUpdater<>::create( - m_local_ioctx, m_threads.get()); + m_local_ioctx, m_threads.get(), ""); C_SaferCond status_updater_ctx; m_local_status_updater->init(&status_updater_ctx); EXPECT_EQ(0, status_updater_ctx.wait()); diff --git a/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc b/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc index b9dd01b5da1..ae379b60bb2 100644 --- a/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc +++ b/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc @@ -167,7 +167,8 @@ public: } void expect_mirror_status_update( - const MirrorImageSiteStatuses& mirror_image_site_statuses, int r) { + const MirrorImageSiteStatuses& mirror_image_site_statuses, + const std::string& site_name, const std::string& fsid, int r) { EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _)) .WillOnce(Invoke([this](auto&&... args) { int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...); @@ -175,8 +176,21 @@ public: return r; })); - for (auto& [global_image_id, mirror_image_status] : + if (!site_name.empty()) { + // status updates to remote site include ping + bufferlist in_bl; + encode(site_name, in_bl); + encode(fsid, in_bl); + encode(static_cast(cls::rbd::MIRROR_PEER_DIRECTION_TX), in_bl); + EXPECT_CALL(*m_mock_local_io_ctx, + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_peer_ping"), ContentsEqual(in_bl), _, _)) + .WillOnce(Return(0)); + } + + for (auto [global_image_id, mirror_image_status] : mirror_image_site_statuses) { + mirror_image_status.fsid = fsid; expect_mirror_status_update(global_image_id, mirror_image_status, r); if (r < 0) { break; @@ -233,7 +247,7 @@ public: TEST_F(TestMockMirrorStatusUpdater, InitShutDown) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, - m_mock_threads); + m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = new MockMirrorStatusWatcher(); @@ -247,7 +261,7 @@ TEST_F(TestMockMirrorStatusUpdater, InitShutDown) { TEST_F(TestMockMirrorStatusUpdater, InitStatusWatcherError) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, - m_mock_threads); + m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = new MockMirrorStatusWatcher(); @@ -264,7 +278,7 @@ TEST_F(TestMockMirrorStatusUpdater, InitStatusWatcherError) { TEST_F(TestMockMirrorStatusUpdater, ShutDownStatusWatcherError) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, - m_mock_threads); + m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = new MockMirrorStatusWatcher(); @@ -283,7 +297,7 @@ TEST_F(TestMockMirrorStatusUpdater, ShutDownStatusWatcherError) { TEST_F(TestMockMirrorStatusUpdater, SmallBatch) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, - m_mock_threads); + m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = new MockMirrorStatusWatcher(); @@ -305,7 +319,7 @@ TEST_F(TestMockMirrorStatusUpdater, SmallBatch) { Context* update_task = nullptr; fire_timer_event(&timer_event, &update_task); - expect_mirror_status_update(mirror_image_site_statuses, 0); + expect_mirror_status_update(mirror_image_site_statuses, "", "", 0); update_task->complete(0); shut_down_mirror_status_updater(mock_mirror_status_updater, @@ -314,7 +328,7 @@ TEST_F(TestMockMirrorStatusUpdater, SmallBatch) { TEST_F(TestMockMirrorStatusUpdater, LargeBatch) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, - m_mock_threads); + m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = new MockMirrorStatusWatcher(); @@ -345,8 +359,8 @@ TEST_F(TestMockMirrorStatusUpdater, LargeBatch) { Context* update_task = nullptr; fire_timer_event(&timer_event, &update_task); - expect_mirror_status_update(mirror_image_site_statuses_1, 0); - expect_mirror_status_update(mirror_image_site_statuses_2, 0); + expect_mirror_status_update(mirror_image_site_statuses_1, "", "", 0); + expect_mirror_status_update(mirror_image_site_statuses_2, "", "", 0); update_task->complete(0); shut_down_mirror_status_updater(mock_mirror_status_updater, @@ -355,7 +369,7 @@ TEST_F(TestMockMirrorStatusUpdater, LargeBatch) { TEST_F(TestMockMirrorStatusUpdater, OverwriteStatus) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, - m_mock_threads); + m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = new MockMirrorStatusWatcher(); @@ -375,7 +389,8 @@ TEST_F(TestMockMirrorStatusUpdater, OverwriteStatus) { expect_mirror_status_update( {{"1", cls::rbd::MirrorImageSiteStatus{ - "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}}, 0); + "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}}, + "", "", 0); update_task->complete(0); shut_down_mirror_status_updater(mock_mirror_status_updater, @@ -384,7 +399,7 @@ TEST_F(TestMockMirrorStatusUpdater, OverwriteStatus) { TEST_F(TestMockMirrorStatusUpdater, OverwriteStatusInFlight) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, - m_mock_threads); + m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = new MockMirrorStatusWatcher(); @@ -414,7 +429,8 @@ TEST_F(TestMockMirrorStatusUpdater, OverwriteStatusInFlight) { expect_work_queue(false); expect_mirror_status_update( {{"1", cls::rbd::MirrorImageSiteStatus{ - "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}}, 0); + "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}}, + "", "", 0); update_task->complete(0); @@ -424,7 +440,7 @@ TEST_F(TestMockMirrorStatusUpdater, OverwriteStatusInFlight) { TEST_F(TestMockMirrorStatusUpdater, ImmediateUpdate) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, - m_mock_threads); + m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = new MockMirrorStatusWatcher(); @@ -435,7 +451,8 @@ TEST_F(TestMockMirrorStatusUpdater, ImmediateUpdate) { *mock_mirror_status_watcher, &timer_event); expect_work_queue(false); - expect_mirror_status_update({{"1", cls::rbd::MirrorImageSiteStatus{}}}, 0); + expect_mirror_status_update({{"1", cls::rbd::MirrorImageSiteStatus{}}}, + "", "", 0); mock_mirror_status_updater.set_mirror_image_status("1", {}, true); shut_down_mirror_status_updater(mock_mirror_status_updater, @@ -444,7 +461,7 @@ TEST_F(TestMockMirrorStatusUpdater, ImmediateUpdate) { TEST_F(TestMockMirrorStatusUpdater, RemoveIdleStatus) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, - m_mock_threads); + m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = new MockMirrorStatusWatcher(); @@ -467,7 +484,7 @@ TEST_F(TestMockMirrorStatusUpdater, RemoveIdleStatus) { TEST_F(TestMockMirrorStatusUpdater, RemoveInFlightStatus) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, - m_mock_threads); + m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = new MockMirrorStatusWatcher(); @@ -501,7 +518,7 @@ TEST_F(TestMockMirrorStatusUpdater, RemoveInFlightStatus) { TEST_F(TestMockMirrorStatusUpdater, ShutDownWhileUpdating) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, - m_mock_threads); + m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = new MockMirrorStatusWatcher(); @@ -535,5 +552,39 @@ TEST_F(TestMockMirrorStatusUpdater, ShutDownWhileUpdating) { ASSERT_EQ(0, on_shutdown.wait()); } +TEST_F(TestMockMirrorStatusUpdater, MirrorPeerSitePing) { + std::string fsid; + ASSERT_EQ(0, _rados->cluster_fsid(&fsid)); + + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, "siteA"); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + MirrorImageSiteStatuses mirror_image_site_statuses; + for (auto i = 0; i < 100; ++i) { + auto pair = mirror_image_site_statuses.emplace( + stringify(i), cls::rbd::MirrorImageSiteStatus{}); + mock_mirror_status_updater.set_mirror_image_status(pair.first->first, + pair.first->second, + false); + } + + Context* update_task = nullptr; + fire_timer_event(&timer_event, &update_task); + + expect_mirror_status_update(mirror_image_site_statuses, "siteA", fsid, 0); + update_task->complete(0); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + } // namespace mirror } // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc index bd52f921814..1dcf61abb45 100644 --- a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc @@ -174,7 +174,8 @@ struct MirrorStatusUpdater { static MirrorStatusUpdater* s_instance; static MirrorStatusUpdater *create(librados::IoCtx &io_ctx, - Threads *threads) { + Threads *threads, + const std::string& site_name) { ceph_assert(s_instance != nullptr); return s_instance; } diff --git a/src/tools/rbd_mirror/MirrorStatusUpdater.cc b/src/tools/rbd_mirror/MirrorStatusUpdater.cc index 43bfdf4ca97..0fd042e828d 100644 --- a/src/tools/rbd_mirror/MirrorStatusUpdater.cc +++ b/src/tools/rbd_mirror/MirrorStatusUpdater.cc @@ -30,11 +30,13 @@ using librbd::util::create_rados_callback; template MirrorStatusUpdater::MirrorStatusUpdater( - librados::IoCtx& io_ctx, Threads *threads) - : m_io_ctx(io_ctx), m_threads(threads), + librados::IoCtx& io_ctx, Threads *threads, + const std::string& site_name) + : m_io_ctx(io_ctx), m_threads(threads), m_site_name(site_name), m_lock(ceph::make_mutex("rbd::mirror::MirrorStatusUpdater " + stringify(m_io_ctx.get_id()))) { - dout(10) << "pool_id=" << m_io_ctx.get_id() << dendl; + dout(10) << "site_name=" << site_name << ", " + << "pool_id=" << m_io_ctx.get_id() << dendl; } template @@ -47,6 +49,16 @@ template void MirrorStatusUpdater::init(Context* on_finish) { dout(10) << dendl; + if (!m_site_name.empty()) { + librados::Rados rados(m_io_ctx); + int r = rados.cluster_fsid(&m_fsid); + if (r < 0) { + derr << "failed to retrieve fsid: " << cpp_strerror(r) << dendl; + m_threads->work_queue->queue(on_finish, r); + return; + } + } + ceph_assert(!m_initialized); m_initialized = true; @@ -299,6 +311,12 @@ void MirrorStatusUpdater::update_task(int r) { librados::ObjectWriteOperation op; uint32_t op_count = 0; + if (!m_site_name.empty()) { + // updates to remote sites should include local site name + // to ensure status includes this peer + librbd::cls_client::mirror_peer_ping(&op, m_site_name, m_fsid); + } + while (it != updating_global_image_ids.end() && op_count < MAX_UPDATES_PER_OP) { auto& global_image_id = *it; @@ -309,6 +327,7 @@ void MirrorStatusUpdater::update_task(int r) { continue; } + status_it->second.fsid = m_fsid; librbd::cls_client::mirror_image_status_set(&op, global_image_id, status_it->second); ++op_count; diff --git a/src/tools/rbd_mirror/MirrorStatusUpdater.h b/src/tools/rbd_mirror/MirrorStatusUpdater.h index 2f23a15fde0..90e4697953c 100644 --- a/src/tools/rbd_mirror/MirrorStatusUpdater.h +++ b/src/tools/rbd_mirror/MirrorStatusUpdater.h @@ -26,11 +26,13 @@ class MirrorStatusUpdater { public: static MirrorStatusUpdater* create(librados::IoCtx& io_ctx, - Threads *threads) { - return new MirrorStatusUpdater(io_ctx, threads); + Threads *threads, + const std::string& site_name) { + return new MirrorStatusUpdater(io_ctx, threads, site_name); } - MirrorStatusUpdater(librados::IoCtx& io_ctx, Threads *threads); + MirrorStatusUpdater(librados::IoCtx& io_ctx, Threads *threads, + const std::string& site_name); ~MirrorStatusUpdater(); void init(Context* on_finish); @@ -68,6 +70,8 @@ private: librados::IoCtx m_io_ctx; Threads* m_threads; + std::string m_site_name; + std::string m_fsid; Context* m_timer_task = nullptr; diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc index 8919945cad1..d3910c105b1 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.cc +++ b/src/tools/rbd_mirror/NamespaceReplayer.cc @@ -265,7 +265,7 @@ void NamespaceReplayer::init_local_status_updater() { ceph_assert(!m_local_status_updater); m_local_status_updater.reset(MirrorStatusUpdater::create( - m_local_io_ctx, m_threads)); + m_local_io_ctx, m_threads, "")); auto ctx = create_context_callback< NamespaceReplayer, &NamespaceReplayer::handle_init_local_status_updater>(this);