From 61a43fe7c2a9fa772517123489ed1fc457197db0 Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Mon, 12 Oct 2020 10:25:46 +0100 Subject: [PATCH] librbd: async API for creating mirror snapshots Signed-off-by: Mykola Golub (cherry picked from commit d8468ec7e49d93860c3b8b06ff36bde539d10a94) Conflicts: src/librbd/api/Mirror.h/cc: no quiesce flags src/test/rbd_mirror/test_ImageReplayer.cc: no quiesce flags --- src/include/rbd/librbd.h | 5 ++ src/include/rbd/librbd.hpp | 2 + src/librbd/api/Mirror.cc | 92 +++++++++++++++++------ src/librbd/api/Mirror.h | 5 +- src/librbd/librbd.cc | 29 ++++++- src/test/librbd/test_mirroring.cc | 63 ++++++++++++++++ src/test/rbd_mirror/test_ImageReplayer.cc | 2 +- 7 files changed, 171 insertions(+), 27 deletions(-) diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index 495e904a697fc..f964bf0c89bc2 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -1251,6 +1251,11 @@ CEPH_RBD_API int rbd_aio_mirror_image_get_status( size_t status_size, rbd_completion_t c) CEPH_RBD_DEPRECATED; +CEPH_RBD_API int rbd_aio_mirror_image_create_snapshot(rbd_image_t image, + uint32_t flags, + uint64_t *snap_id, + rbd_completion_t c); + // RBD groups support functions CEPH_RBD_API int rbd_group_create(rados_ioctx_t p, const char *name); CEPH_RBD_API int rbd_group_remove(rados_ioctx_t p, const char *name); diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp index 1bb74ce72f2e9..3f5f63263b1cd 100644 --- a/src/include/rbd/librbd.hpp +++ b/src/include/rbd/librbd.hpp @@ -768,6 +768,8 @@ public: mirror_image_status_t *mirror_image_status, size_t status_size, RBD::AioCompletion *c) CEPH_RBD_DEPRECATED; + int aio_mirror_image_create_snapshot(uint32_t flags, uint64_t *snap_id, + RBD::AioCompletion *c); int update_watch(UpdateWatchCtx *ctx, uint64_t *handle); int update_unwatch(uint64_t handle); diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index ce92b85427441..0a868bf976144 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -35,6 +35,7 @@ #include "json_spirit/json_spirit.h" #include +#include #define dout_subsys ceph_subsys_rbd #undef dout_prefix @@ -353,6 +354,39 @@ struct C_ImageGetGlobalStatus : public C_ImageGetInfo { } }; +template +struct C_ImageSnapshotCreate : public Context { + I *ictx; + uint64_t *snap_id; + Context *on_finish; + + cls::rbd::MirrorImage mirror_image; + mirror::PromotionState promotion_state; + std::string primary_mirror_uuid; + + C_ImageSnapshotCreate(I *ictx, uint64_t *snap_id, Context *on_finish) + : ictx(ictx), snap_id(snap_id), on_finish(on_finish) { + } + + void finish(int r) override { + if (r < 0 && r != -ENOENT) { + on_finish->complete(r); + return; + } + + if (mirror_image.mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT || + mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + lderr(ictx->cct) << "snapshot based mirroring is not enabled" << dendl; + on_finish->complete(-EINVAL); + return; + } + + auto req = mirror::snapshot::CreatePrimaryRequest::create( + ictx, mirror_image.global_image_id, CEPH_NOSNAP, 0U, snap_id, on_finish); + req->send(); + } +}; + } // anonymous namespace template @@ -1970,36 +2004,48 @@ int Mirror::image_info_list( } template -int Mirror::image_snapshot_create(I *ictx, uint64_t *snap_id) { +int Mirror::image_snapshot_create(I *ictx, uint32_t flags, + uint64_t *snap_id) { + C_SaferCond ctx; + Mirror::image_snapshot_create(ictx, flags, snap_id, &ctx); + + return ctx.wait(); +} + +template +void Mirror::image_snapshot_create(I *ictx, uint32_t flags, + uint64_t *snap_id, Context *on_finish) { CephContext *cct = ictx->cct; ldout(cct, 20) << "ictx=" << ictx << dendl; - int r = ictx->state->refresh_if_required(); - if (r < 0) { - return r; + if (flags != 0U) { + lderr(cct) << "invalid snap create flags: " << std::bitset<32>(flags) + << dendl; + on_finish->complete(-EINVAL); + return; } - cls::rbd::MirrorImage mirror_image; - r = cls_client::mirror_image_get(&ictx->md_ctx, ictx->id, - &mirror_image); - if (r == -ENOENT) { - return -EINVAL; - } else if (r < 0) { - lderr(cct) << "failed to retrieve mirror image" << dendl; - return r; - } + auto on_refresh = new LambdaContext( + [ictx, snap_id, on_finish](int r) { + if (r < 0) { + lderr(ictx->cct) << "refresh failed: " << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } - if (mirror_image.mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT || - mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { - lderr(cct) << "snapshot based mirroring is not enabled" << dendl; - return -EINVAL; - } + auto ctx = new C_ImageSnapshotCreate(ictx, snap_id, on_finish); + auto req = mirror::GetInfoRequest::create(*ictx, &ctx->mirror_image, + &ctx->promotion_state, + &ctx->primary_mirror_uuid, + ctx); + req->send(); + }); - C_SaferCond on_finish; - auto req = mirror::snapshot::CreatePrimaryRequest::create( - ictx, mirror_image.global_image_id, CEPH_NOSNAP, 0U, snap_id, &on_finish); - req->send(); - return on_finish.wait(); + if (ictx->state->is_refresh_required()) { + ictx->state->refresh(on_refresh); + } else { + on_refresh->complete(0); + } } } // namespace api diff --git a/src/librbd/api/Mirror.h b/src/librbd/api/Mirror.h index b0dae99bc4a1e..c4cc901d2d1ae 100644 --- a/src/librbd/api/Mirror.h +++ b/src/librbd/api/Mirror.h @@ -113,7 +113,10 @@ struct Mirror { Context *on_finish); static int image_get_instance_id(ImageCtxT *ictx, std::string *instance_id); - static int image_snapshot_create(ImageCtxT *ictx, uint64_t *snap_id); + static int image_snapshot_create(ImageCtxT *ictx, uint32_t flags, + uint64_t *snap_id); + static void image_snapshot_create(ImageCtxT *ictx, uint32_t flags, + uint64_t *snap_id, Context *on_finish); }; } // namespace api diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index 9a3cc3e67d1aa..438b587168375 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -2834,7 +2834,7 @@ namespace librbd { int Image::mirror_image_create_snapshot(uint64_t *snap_id) { ImageCtx *ictx = (ImageCtx *)ctx; - return librbd::api::Mirror<>::image_snapshot_create(ictx, snap_id); + return librbd::api::Mirror<>::image_snapshot_create(ictx, 0U, snap_id); } int Image::mirror_image_get_info(mirror_image_info_t *mirror_image_info, @@ -2986,6 +2986,17 @@ namespace librbd { #pragma GCC diagnostic pop + int Image::aio_mirror_image_create_snapshot(uint32_t flags, uint64_t *snap_id, + RBD::AioCompletion *c) { + ImageCtx *ictx = (ImageCtx *)ctx; + + librbd::api::Mirror<>::image_snapshot_create( + ictx, flags, snap_id, new C_AioCompletion(ictx, + librbd::io::AIO_TYPE_GENERIC, + get_aio_completion(c))); + return 0; + } + int Image::update_watch(UpdateWatchCtx *wctx, uint64_t *handle) { ImageCtx *ictx = (ImageCtx *)ctx; tracepoint(librbd, update_watch_enter, ictx, wctx); @@ -6301,7 +6312,7 @@ extern "C" int rbd_mirror_image_create_snapshot(rbd_image_t image, uint64_t *snap_id) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; - return librbd::api::Mirror<>::image_snapshot_create(ictx, snap_id); + return librbd::api::Mirror<>::image_snapshot_create(ictx, 0U, snap_id); } extern "C" int rbd_mirror_image_get_info(rbd_image_t image, @@ -6497,6 +6508,20 @@ extern "C" int rbd_aio_mirror_image_get_status( #pragma GCC diagnostic pop +extern "C" int rbd_aio_mirror_image_create_snapshot(rbd_image_t image, + uint32_t flags, + uint64_t *snap_id, + rbd_completion_t c) { + librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; + librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c; + + librbd::api::Mirror<>::image_snapshot_create( + ictx, flags, snap_id, new C_AioCompletion(ictx, + librbd::io::AIO_TYPE_GENERIC, + get_aio_completion(comp))); + return 0; +} + extern "C" int rbd_update_watch(rbd_image_t image, uint64_t *handle, rbd_update_callback_t watch_cb, void *arg) { diff --git a/src/test/librbd/test_mirroring.cc b/src/test/librbd/test_mirroring.cc index a3eacd75747b7..c72c34c32d4b2 100644 --- a/src/test/librbd/test_mirroring.cc +++ b/src/test/librbd/test_mirroring.cc @@ -1471,3 +1471,66 @@ TEST_F(TestMirroring, SnapshotPromoteDemote) ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer_uuid)); ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED)); } + +TEST_F(TestMirroring, AioSnapshotCreate) +{ + REQUIRE_FORMAT_V2(); + + std::list image_names; + for (size_t idx = 0; idx < 10; ++idx) { + image_names.push_back(get_temp_image_name()); + } + + ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE)); + std::string peer_uuid; + ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer_uuid, + RBD_MIRROR_PEER_DIRECTION_RX_TX, + "cluster", "client")); + // create mirror images + uint64_t features; + ASSERT_TRUE(get_features(&features)); + int order = 20; + std::list images; + for (auto &image_name : image_names) { + ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 2048, features, + &order)); + images.emplace_back(); + ASSERT_EQ(0, m_rbd.open(m_ioctx, images.back(), image_name.c_str())); + ASSERT_EQ(0, images.back().mirror_image_enable2( + RBD_MIRROR_IMAGE_MODE_SNAPSHOT)); + } + + // create snapshots + std::list snap_ids; + std::list aio_comps; + for (auto &image : images) { + snap_ids.emplace_back(); + aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr)); + ASSERT_EQ(0, image.aio_mirror_image_create_snapshot(0, &snap_ids.back(), + aio_comps.back())); + } + for (auto aio_comp : aio_comps) { + ASSERT_EQ(0, aio_comp->wait_for_complete()); + ASSERT_EQ(1, aio_comp->is_complete()); + ASSERT_EQ(0, aio_comp->get_return_value()); + aio_comp->release(); + } + aio_comps.clear(); + + // verify + for (auto &image : images) { + vector snaps; + ASSERT_EQ(0, image.snap_list(snaps)); + ASSERT_EQ(2U, snaps.size()); + ASSERT_EQ(snaps[1].id, snap_ids.front()); + + std::string image_name; + ASSERT_EQ(0, image.get_name(&image_name)); + ASSERT_EQ(0, image.close()); + ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str())); + snap_ids.pop_front(); + } + + ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer_uuid)); + ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED)); +} diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index 69dabbb7bcb5d..ed3f9cbca2ac4 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -536,7 +536,7 @@ public: } else { uint64_t snap_id = CEPH_NOSNAP; ASSERT_EQ(0, librbd::api::Mirror<>::image_snapshot_create( - ictx, &snap_id)); + ictx, 0U, &snap_id)); } printf("flushed\n"); -- 2.39.5