From: Arthur Outhenin-Chalandre Date: Tue, 13 Jul 2021 12:19:49 +0000 (+0200) Subject: rbd-mirror: add image_map cleanup in LoadRequest X-Git-Tag: v16.2.7~65^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=838b1b2c607e3123ad61888ad2082d6dcd045595;p=ceph.git rbd-mirror: add image_map cleanup in LoadRequest In the LoadRequest in the ImageMap class add initial cleanup to remove stale entries. To cleanup the LoadRequest will query the mirror image list and remove all the image_map that are notin the list. Signed-off-by: Arthur Outhenin-Chalandre (cherry picked from commit e135403c736295b63fe1c8a861af40de302b8b57) --- diff --git a/qa/workunits/rbd/rbd_mirror_journal.sh b/qa/workunits/rbd/rbd_mirror_journal.sh index 75c60123639b..56a8b13a9a77 100755 --- a/qa/workunits/rbd/rbd_mirror_journal.sh +++ b/qa/workunits/rbd/rbd_mirror_journal.sh @@ -409,11 +409,6 @@ for i in ${image2} ${image4}; do done testlog "TEST: disable mirror while daemon is stopped" -# TODO: workaround for the daemon to ack the deletion, to remove when -# image_map cleanup is fixed -for i in ${image2} ${image4}; do - wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' -done stop_mirrors ${CLUSTER1} stop_mirrors ${CLUSTER2} set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' @@ -422,11 +417,6 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then test_image_present ${CLUSTER1} ${POOL} ${image} 'present' fi start_mirrors ${CLUSTER1} -start_mirrors ${CLUSTER2} # TODO: remove start/stop of cluster2 deamons when - # image_map cleanup at startup is resolved -wait_for_image_in_omap ${CLUSTER1} ${POOL} -wait_for_image_in_omap ${CLUSTER2} ${POOL} -stop_mirrors ${CLUSTER2} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool' enable_journaling ${CLUSTER2} ${POOL} ${image} diff --git a/qa/workunits/rbd/rbd_mirror_snapshot.sh b/qa/workunits/rbd/rbd_mirror_snapshot.sh index f83b166984e1..0060440fb8d0 100755 --- a/qa/workunits/rbd/rbd_mirror_snapshot.sh +++ b/qa/workunits/rbd/rbd_mirror_snapshot.sh @@ -384,11 +384,6 @@ for i in ${image2} ${image4}; do done testlog "TEST: disable mirror while daemon is stopped" -# TODO: workaround for the daemon to ack the deletion, to remove when -# image_map cleanup is fixed -for i in ${image2} ${image4}; do - wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' -done stop_mirrors ${CLUSTER1} stop_mirrors ${CLUSTER2} disable_mirror ${CLUSTER2} ${POOL} ${image} @@ -396,11 +391,6 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then test_image_present ${CLUSTER1} ${POOL} ${image} 'present' fi start_mirrors ${CLUSTER1} -start_mirrors ${CLUSTER2} # TODO: remove start/stop of cluster2 deamons when - # image_map cleanup at startup is resolved -wait_for_image_in_omap ${CLUSTER1} ${POOL} -wait_for_image_in_omap ${CLUSTER2} ${POOL} -stop_mirrors ${CLUSTER2} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' enable_mirror ${CLUSTER2} ${POOL} ${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' diff --git a/src/tools/rbd_mirror/image_map/LoadRequest.cc b/src/tools/rbd_mirror/image_map/LoadRequest.cc index 7387b476cc0f..46564a1607a2 100644 --- a/src/tools/rbd_mirror/image_map/LoadRequest.cc +++ b/src/tools/rbd_mirror/image_map/LoadRequest.cc @@ -8,6 +8,7 @@ #include "include/rbd_types.h" #include "cls/rbd/cls_rbd_client.h" +#include "UpdateRequest.h" #include "LoadRequest.h" #define dout_context g_ceph_context @@ -23,6 +24,7 @@ namespace image_map { static const uint32_t MAX_RETURN = 1024; using librbd::util::create_rados_callback; +using librbd::util::create_context_callback; template LoadRequest::LoadRequest(librados::IoCtx &ioctx, @@ -80,7 +82,81 @@ void LoadRequest::handle_image_map_list(int r) { return; } - finish(0); + mirror_image_list(); +} + +template +void LoadRequest::mirror_image_list() { + dout(20) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_list_start(&op, m_start_after, MAX_RETURN); + + m_out_bl.clear(); + librados::AioCompletion *aio_comp = create_rados_callback< + LoadRequest, + &LoadRequest::handle_mirror_image_list>(this); + int r = m_ioctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void LoadRequest::handle_mirror_image_list(int r) { + dout(20) << ": r=" << r << dendl; + + std::map ids; + if (r == 0) { + auto it = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_image_list_finish(&it, &ids); + } + + if (r < 0 && r != -ENOENT) { + derr << "failed to list mirrored images: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + for (auto &id : ids) { + m_global_image_ids.emplace(id.second); + } + + if (ids.size() == MAX_RETURN) { + m_start_after = ids.rbegin()->first; + mirror_image_list(); + return; + } + + cleanup_image_map(); +} + +template +void LoadRequest::cleanup_image_map() { + dout(20) << dendl; + + std::set map_removals; + + auto it = m_image_mapping->begin(); + while (it != m_image_mapping->end()) { + if (m_global_image_ids.count(it->first) > 0) { + ++it; + continue; + } + map_removals.emplace(it->first); + it = m_image_mapping->erase(it); + } + + if (map_removals.size() == 0) { + finish(0); + return; + } + + auto ctx = create_context_callback< + LoadRequest, + &LoadRequest::finish>(this); + image_map::UpdateRequest *req = image_map::UpdateRequest::create( + m_ioctx, {}, std::move(map_removals), ctx); + req->send(); } template diff --git a/src/tools/rbd_mirror/image_map/LoadRequest.h b/src/tools/rbd_mirror/image_map/LoadRequest.h index 7657e1108a06..9b1be96857ae 100644 --- a/src/tools/rbd_mirror/image_map/LoadRequest.h +++ b/src/tools/rbd_mirror/image_map/LoadRequest.h @@ -36,6 +36,12 @@ private: * IMAGE_MAP_LIST. . . . . . . * | * v + * MIRROR_IMAGE_LIST + * | + * v + * CLEANUP_IMAGE_MAP + * | + * v * * * @endverbatim @@ -48,12 +54,19 @@ private: std::map *m_image_mapping; Context *m_on_finish; + std::set m_global_image_ids; + bufferlist m_out_bl; std::string m_start_after; void image_map_list(); void handle_image_map_list(int r); + void mirror_image_list(); + void handle_mirror_image_list(int r); + + void cleanup_image_map(); + void finish(int r); };