From f096cf5fab86e8748dab5c0c03c9b73c3d996e5f Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Tue, 13 Jul 2021 14:19:49 +0200 Subject: [PATCH] rbd-mirror: add image_map cleanup in LoadRequest In the LoadRequest in the ImageMap class add initial cleanup to remove stale entries. To cleanup the LoadRequest will query the mirror image list and remove all the image_map that are notin the list. Signed-off-by: Arthur Outhenin-Chalandre (cherry picked from commit e135403c736295b63fe1c8a861af40de302b8b57) --- qa/workunits/rbd/rbd_mirror_journal.sh | 10 --- qa/workunits/rbd/rbd_mirror_snapshot.sh | 10 --- src/tools/rbd_mirror/image_map/LoadRequest.cc | 78 ++++++++++++++++++- src/tools/rbd_mirror/image_map/LoadRequest.h | 13 ++++ 4 files changed, 90 insertions(+), 21 deletions(-) diff --git a/qa/workunits/rbd/rbd_mirror_journal.sh b/qa/workunits/rbd/rbd_mirror_journal.sh index 39db7bcb023e0..f878dcc38ccad 100755 --- a/qa/workunits/rbd/rbd_mirror_journal.sh +++ b/qa/workunits/rbd/rbd_mirror_journal.sh @@ -409,11 +409,6 @@ for i in ${image2} ${image4}; do done testlog "TEST: disable mirror while daemon is stopped" -# TODO: workaround for the daemon to ack the deletion, to remove when -# image_map cleanup is fixed -for i in ${image2} ${image4}; do - wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' -done stop_mirrors ${CLUSTER1} stop_mirrors ${CLUSTER2} set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' @@ -422,11 +417,6 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then test_image_present ${CLUSTER1} ${POOL} ${image} 'present' fi start_mirrors ${CLUSTER1} -start_mirrors ${CLUSTER2} # TODO: remove start/stop of cluster2 deamons when - # image_map cleanup at startup is resolved -wait_for_image_in_omap ${CLUSTER1} ${POOL} -wait_for_image_in_omap ${CLUSTER2} ${POOL} -stop_mirrors ${CLUSTER2} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool' enable_journaling ${CLUSTER2} ${POOL} ${image} diff --git a/qa/workunits/rbd/rbd_mirror_snapshot.sh b/qa/workunits/rbd/rbd_mirror_snapshot.sh index 6d3f5846de168..8ce7857cd8827 100755 --- a/qa/workunits/rbd/rbd_mirror_snapshot.sh +++ b/qa/workunits/rbd/rbd_mirror_snapshot.sh @@ -384,11 +384,6 @@ for i in ${image2} ${image4}; do done testlog "TEST: disable mirror while daemon is stopped" -# TODO: workaround for the daemon to ack the deletion, to remove when -# image_map cleanup is fixed -for i in ${image2} ${image4}; do - wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' -done stop_mirrors ${CLUSTER1} stop_mirrors ${CLUSTER2} disable_mirror ${CLUSTER2} ${POOL} ${image} @@ -396,11 +391,6 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then test_image_present ${CLUSTER1} ${POOL} ${image} 'present' fi start_mirrors ${CLUSTER1} -start_mirrors ${CLUSTER2} # TODO: remove start/stop of cluster2 deamons when - # image_map cleanup at startup is resolved -wait_for_image_in_omap ${CLUSTER1} ${POOL} -wait_for_image_in_omap ${CLUSTER2} ${POOL} -stop_mirrors ${CLUSTER2} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' enable_mirror ${CLUSTER2} ${POOL} ${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' diff --git a/src/tools/rbd_mirror/image_map/LoadRequest.cc b/src/tools/rbd_mirror/image_map/LoadRequest.cc index 7387b476cc0ff..46564a1607a2b 100644 --- a/src/tools/rbd_mirror/image_map/LoadRequest.cc +++ b/src/tools/rbd_mirror/image_map/LoadRequest.cc @@ -8,6 +8,7 @@ #include "include/rbd_types.h" #include "cls/rbd/cls_rbd_client.h" +#include "UpdateRequest.h" #include "LoadRequest.h" #define dout_context g_ceph_context @@ -23,6 +24,7 @@ namespace image_map { static const uint32_t MAX_RETURN = 1024; using librbd::util::create_rados_callback; +using librbd::util::create_context_callback; template LoadRequest::LoadRequest(librados::IoCtx &ioctx, @@ -80,7 +82,81 @@ void LoadRequest::handle_image_map_list(int r) { return; } - finish(0); + mirror_image_list(); +} + +template +void LoadRequest::mirror_image_list() { + dout(20) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_list_start(&op, m_start_after, MAX_RETURN); + + m_out_bl.clear(); + librados::AioCompletion *aio_comp = create_rados_callback< + LoadRequest, + &LoadRequest::handle_mirror_image_list>(this); + int r = m_ioctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void LoadRequest::handle_mirror_image_list(int r) { + dout(20) << ": r=" << r << dendl; + + std::map ids; + if (r == 0) { + auto it = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_image_list_finish(&it, &ids); + } + + if (r < 0 && r != -ENOENT) { + derr << "failed to list mirrored images: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + for (auto &id : ids) { + m_global_image_ids.emplace(id.second); + } + + if (ids.size() == MAX_RETURN) { + m_start_after = ids.rbegin()->first; + mirror_image_list(); + return; + } + + cleanup_image_map(); +} + +template +void LoadRequest::cleanup_image_map() { + dout(20) << dendl; + + std::set map_removals; + + auto it = m_image_mapping->begin(); + while (it != m_image_mapping->end()) { + if (m_global_image_ids.count(it->first) > 0) { + ++it; + continue; + } + map_removals.emplace(it->first); + it = m_image_mapping->erase(it); + } + + if (map_removals.size() == 0) { + finish(0); + return; + } + + auto ctx = create_context_callback< + LoadRequest, + &LoadRequest::finish>(this); + image_map::UpdateRequest *req = image_map::UpdateRequest::create( + m_ioctx, {}, std::move(map_removals), ctx); + req->send(); } template diff --git a/src/tools/rbd_mirror/image_map/LoadRequest.h b/src/tools/rbd_mirror/image_map/LoadRequest.h index 7657e1108a068..9b1be96857aeb 100644 --- a/src/tools/rbd_mirror/image_map/LoadRequest.h +++ b/src/tools/rbd_mirror/image_map/LoadRequest.h @@ -36,6 +36,12 @@ private: * IMAGE_MAP_LIST. . . . . . . * | * v + * MIRROR_IMAGE_LIST + * | + * v + * CLEANUP_IMAGE_MAP + * | + * v * * * @endverbatim @@ -48,12 +54,19 @@ private: std::map *m_image_mapping; Context *m_on_finish; + std::set m_global_image_ids; + bufferlist m_out_bl; std::string m_start_after; void image_map_list(); void handle_image_map_list(int r); + void mirror_image_list(); + void handle_mirror_image_list(int r); + + void cleanup_image_map(); + void finish(int r); }; -- 2.39.5