From: Mykola Golub Date: Thu, 21 Jan 2021 09:18:36 +0000 (+0000) Subject: rbd-mirror: make pool watcher also refresh groups X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=5e6c52fe39846b740a6945a0bee1888508d2289b;p=ceph-ci.git rbd-mirror: make pool watcher also refresh groups Signed-off-by: Mykola Golub Signed-off-by: Prasanna Kumar Kalever --- diff --git a/src/test/rbd_mirror/pool_watcher/test_mock_RefreshEntitiesRequest.cc b/src/test/rbd_mirror/pool_watcher/test_mock_RefreshEntitiesRequest.cc index bf4e9493f8a..05c9646381a 100644 --- a/src/test/rbd_mirror/pool_watcher/test_mock_RefreshEntitiesRequest.cc +++ b/src/test/rbd_mirror/pool_watcher/test_mock_RefreshEntitiesRequest.cc @@ -55,11 +55,45 @@ public: Return(r))); } + void expect_mirror_group_list( + librados::IoCtx &io_ctx, + const std::map &groups, int r) { + bufferlist bl; + encode(groups, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_group_list"), + _, _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_group_image_list( + librados::IoCtx &io_ctx, const std::string &group_id, + const std::vector &images, int r) { + bufferlist bl; + encode(images, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(librbd::util::group_header_name(group_id), _, StrEq("rbd"), + StrEq("group_image_list"), _, _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } }; TEST_F(TestMockPoolWatcherRefreshEntitiesRequest, Success) { InSequence seq; expect_mirror_image_list(m_remote_io_ctx, {{"local id", "global id"}}, 0); + cls::rbd::MirrorGroup mirror_group = + {"global id", cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + cls::rbd::MIRROR_GROUP_STATE_ENABLED}; + expect_mirror_group_list(m_remote_io_ctx, {{"local id", mirror_group}}, 0); + expect_group_image_list(m_remote_io_ctx, "local id", {{}, {}, {}}, 0); C_SaferCond ctx; std::map entities; @@ -70,25 +104,50 @@ TEST_F(TestMockPoolWatcherRefreshEntitiesRequest, Success) { ASSERT_EQ(0, ctx.wait()); std::map expected_entities = - {{{MIRROR_ENTITY_TYPE_IMAGE, "global id", 1}, "local id"}}; + {{{MIRROR_ENTITY_TYPE_IMAGE, "global id", 1}, "local id"}, + {{MIRROR_ENTITY_TYPE_GROUP, "global id", 3}, "local id"}}; ASSERT_EQ(expected_entities, entities); } TEST_F(TestMockPoolWatcherRefreshEntitiesRequest, LargeDirectory) { InSequence seq; - std::map mirror_list; + std::map mirror_image_list; std::map expected_entities; for (uint32_t idx = 1; idx <= 1024; ++idx) { - mirror_list.insert(std::make_pair("local id " + stringify(idx), - "global id " + stringify(idx))); + mirror_image_list.insert({"local id " + stringify(idx), + "global id " + stringify(idx)}); expected_entities.insert( {{MIRROR_ENTITY_TYPE_IMAGE, "global id " + stringify(idx), 1}, "local id " + stringify(idx)}); } - expect_mirror_image_list(m_remote_io_ctx, mirror_list, 0); + expect_mirror_image_list(m_remote_io_ctx, mirror_image_list, 0); expect_mirror_image_list(m_remote_io_ctx, {{"local id", "global id"}}, 0); + std::map mirror_group_list; + for (uint32_t idx = 1; idx <= 1024; ++idx) { + cls::rbd::MirrorGroup mirror_group = + {"global id " + stringify(idx), + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + cls::rbd::MIRROR_GROUP_STATE_ENABLED}; + + mirror_group_list.insert({"local id " + stringify(idx), mirror_group}); + expected_entities.insert( + {{MIRROR_ENTITY_TYPE_GROUP, "global id " + stringify(idx), 2}, + "local id " + stringify(idx)}); + } + + expect_mirror_group_list(m_remote_io_ctx, mirror_group_list, 0); + cls::rbd::MirrorGroup mirror_group = + {"global id", cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + cls::rbd::MIRROR_GROUP_STATE_ENABLED}; + expect_mirror_group_list(m_remote_io_ctx, {{"local id", mirror_group}}, 0); + + expect_group_image_list(m_remote_io_ctx, "local id", {{}, {}}, 0); + for (auto &[group_id, _] : mirror_group_list) { + expect_group_image_list(m_remote_io_ctx, group_id, {{}, {}}, 0); + } + C_SaferCond ctx; std::map entities; MockRefreshEntitiesRequest *req = new MockRefreshEntitiesRequest( @@ -99,6 +158,8 @@ TEST_F(TestMockPoolWatcherRefreshEntitiesRequest, LargeDirectory) { expected_entities.insert( {{MIRROR_ENTITY_TYPE_IMAGE, "global id", 1}, "local id"}); + expected_entities.insert( + {{MIRROR_ENTITY_TYPE_GROUP, "global id", 2}, "local id"}); ASSERT_EQ(expected_entities, entities); } diff --git a/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.cc b/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.cc index 25d84d4bfb3..78f0966cf65 100644 --- a/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.cc +++ b/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.cc @@ -71,7 +71,111 @@ void RefreshEntitiesRequest::handle_mirror_image_list(int r) { return; } - finish(0); + m_start_after = {}; + mirror_group_list(); +} + +template +void RefreshEntitiesRequest::mirror_group_list() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_group_list_start(&op, m_start_after, MAX_RETURN); + + m_out_bl.clear(); + librados::AioCompletion *aio_comp = create_rados_callback< + RefreshEntitiesRequest, + &RefreshEntitiesRequest::handle_mirror_group_list>(this); + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void RefreshEntitiesRequest::handle_mirror_group_list(int r) { + dout(10) << "r=" << r << dendl; + + std::map groups; + if (r == 0) { + auto it = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_group_list_finish(&it, &groups); + } + + if (r < 0 && r != -ENOENT) { + derr << "failed to list mirrored groups: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_groups.insert(groups.begin(), groups.end()); + + if (groups.size() == MAX_RETURN) { + m_start_after = groups.rbegin()->first; + mirror_group_list(); + return; + } + + // XXXMG: should we just provide group_size field in cls::rbd::MirrorGroup + // instead of listing group images just to learn their count? + group_image_list(); +} + +template +void RefreshEntitiesRequest::group_image_list() { + if (m_groups.empty()) { + finish(0); + return; + } + + auto &group_id = m_groups.begin()->first; + + dout(10) << group_id << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::group_image_list_start( + &op, m_start_group_image_list_after, MAX_RETURN); + m_out_bl.clear(); + librados::AioCompletion *aio_comp = create_rados_callback< + RefreshEntitiesRequest, + &RefreshEntitiesRequest::handle_group_image_list>(this); + int r = m_remote_io_ctx.aio_operate(librbd::util::group_header_name(group_id), + aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void RefreshEntitiesRequest::handle_group_image_list(int r) { + dout(10) << "r=" << r << dendl; + + std::vector images; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::group_image_list_finish(&iter, &images); + } + + if (r < 0) { + derr << "error listing remote group: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + auto image_count = images.size(); + m_group_size += image_count; + if (image_count == MAX_RETURN) { + m_start_group_image_list_after = images.rbegin()->spec; + } else { + auto group_it = m_groups.begin(); + auto &global_group_id = group_it->second.global_group_id; + m_entities->insert( + {{MIRROR_ENTITY_TYPE_GROUP, global_group_id, m_group_size}, + group_it->first}); + m_groups.erase(group_it); + m_start_group_image_list_after = {}; + m_group_size = 0; + } + + group_image_list(); } template diff --git a/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.h b/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.h index ab1957a82b0..70a5bda5ec5 100644 --- a/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.h +++ b/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.h @@ -6,6 +6,7 @@ #include "include/buffer.h" #include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_types.h" #include "tools/rbd_mirror/Types.h" #include @@ -48,6 +49,16 @@ private: * v v | (more images) * MIRROR_IMAGE_LIST ---/ * | + * | /-------------\ + * | | | + * v v | (more groups) + * MIRROR_GROUP_LIST ---/ + * | + * | /-------------\ + * | | | + * v v | (for every group) + * GROUP_IMAGE_LIST ----/ + * | * v * * @@ -61,9 +72,19 @@ private: bufferlist m_out_bl; std::string m_start_after; + std::map m_groups; + cls::rbd::GroupImageSpec m_start_group_image_list_after; + size_t m_group_size = 0; + void mirror_image_list(); void handle_mirror_image_list(int r); + void mirror_group_list(); + void handle_mirror_group_list(int r); + + void group_image_list(); + void handle_group_image_list(int r); + void finish(int r); };