From 680e86e6fbf4372e5728a18d57728d251b29ba3d Mon Sep 17 00:00:00 2001 From: songweibin Date: Tue, 31 Jul 2018 16:53:24 +0800 Subject: [PATCH] librbd: add group snap rollback method Fixes: http://tracker.ceph.com/issues/23550 Signed-off-by: songweibin --- src/cls/rbd/cls_rbd_client.cc | 3 +- src/cls/rbd/cls_rbd_client.h | 2 +- src/include/rbd/librbd.h | 8 ++ src/include/rbd/librbd.hpp | 5 ++ src/librbd/api/Group.cc | 160 +++++++++++++++++++++++++++++++++ src/librbd/api/Group.h | 3 + src/librbd/librbd.cc | 70 +++++++++++++++ src/test/librbd/test_Groups.cc | 46 ++++++++++ src/tracing/librbd.tp | 22 +++++ 9 files changed, 317 insertions(+), 2 deletions(-) diff --git a/src/cls/rbd/cls_rbd_client.cc b/src/cls/rbd/cls_rbd_client.cc index 5e6f3181d9d..14123b92bdb 100644 --- a/src/cls/rbd/cls_rbd_client.cc +++ b/src/cls/rbd/cls_rbd_client.cc @@ -2381,7 +2381,7 @@ namespace librbd { op->exec("rbd", "mirror_image_map_remove", bl); } - // Consistency groups functions + // Groups functions int group_dir_list(librados::IoCtx *ioctx, const std::string &oid, const std::string &start, uint64_t max_return, map *cgs) @@ -2567,6 +2567,7 @@ namespace librbd { return 0; } + int group_snap_list(librados::IoCtx *ioctx, const std::string &oid, const cls::rbd::GroupSnapshot &start, uint64_t max_return, diff --git a/src/cls/rbd/cls_rbd_client.h b/src/cls/rbd/cls_rbd_client.h index c9c362bb536..ad3050516bc 100644 --- a/src/cls/rbd/cls_rbd_client.h +++ b/src/cls/rbd/cls_rbd_client.h @@ -505,7 +505,7 @@ namespace librbd { void mirror_image_map_remove(librados::ObjectWriteOperation *op, const std::string &global_image_id); - // Consistency groups functions + // Groups functions int group_dir_list(librados::IoCtx *ioctx, const std::string &oid, const std::string &start, uint64_t max_return, map *groups); diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index 85e9636f426..aaad6b811af 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -1038,6 +1038,14 @@ CEPH_RBD_API int rbd_group_snap_list(rados_ioctx_t group_p, CEPH_RBD_API int rbd_group_snap_list_cleanup(rbd_group_snap_info_t *snaps, size_t group_snap_info_size, size_t num_entries); +CEPH_RBD_API int rbd_group_snap_rollback(rados_ioctx_t group_p, + const char *group_name, + const char *snap_name); +CEPH_RBD_API int rbd_group_snap_rollback_with_progress(rados_ioctx_t group_p, + const char *group_name, + const char *snap_name, + librbd_progress_fn_t cb, + void *cbdata); CEPH_RBD_API int rbd_namespace_create(rados_ioctx_t io, const char *namespace_name); diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp index 38cc707a31f..052ec111caf 100644 --- a/src/include/rbd/librbd.hpp +++ b/src/include/rbd/librbd.hpp @@ -267,6 +267,11 @@ public: int group_snap_list(IoCtx& group_ioctx, const char *group_name, std::vector *snaps, size_t group_snap_info_size); + int group_snap_rollback(IoCtx& io_ctx, const char *group_name, + const char *snap_name); + int group_snap_rollback_with_progress(IoCtx& io_ctx, const char *group_name, + const char *snap_name, + ProgressContext& pctx); int namespace_create(IoCtx& ioctx, const char *namespace_name); int namespace_remove(IoCtx& ioctx, const char *namespace_name); diff --git a/src/librbd/api/Group.cc b/src/librbd/api/Group.cc index c0540881b31..6dd971037da 100644 --- a/src/librbd/api/Group.cc +++ b/src/librbd/api/Group.cc @@ -313,6 +313,127 @@ finish: return ret_code; } +int group_snap_rollback_by_record(librados::IoCtx& group_ioctx, + const cls::rbd::GroupSnapshot& group_snap, + const std::string& group_id, + const std::string& group_header_oid, + ProgressContext& pctx) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + librados::Rados rados(group_ioctx); + std::vector on_finishes; + int r, ret_code; + + std::vector ictxs; + + cls::rbd::GroupSnapshotNamespace ne{group_ioctx.get_id(), group_id, + group_snap.id}; + + ldout(cct, 20) << "Rolling back snapshots" << dendl; + int snap_count = group_snap.snaps.size(); + + for (int i = 0; i < snap_count; ++i) { + librados::IoCtx image_io_ctx; + r = rados.ioctx_create2(group_snap.snaps[i].pool, image_io_ctx); + if (r < 0) { + ldout(cct, 1) << "Failed to create io context for image" << dendl; + } + + librbd::ImageCtx* image_ctx = new ImageCtx("", group_snap.snaps[i].image_id, + nullptr, image_io_ctx, false); + + C_SaferCond* on_finish = new C_SaferCond; + + image_ctx->state->open(false, on_finish); + + ictxs.push_back(image_ctx); + on_finishes.push_back(on_finish); + } + + ret_code = 0; + for (int i = 0; i < snap_count; ++i) { + r = on_finishes[i]->wait(); + delete on_finishes[i]; + if (r < 0) { + delete ictxs[i]; + ictxs[i] = nullptr; + ret_code = r; + } + } + if (ret_code != 0) { + goto finish; + } + + ldout(cct, 20) << "Requesting exclusive locks for images" << dendl; + for (auto ictx: ictxs) { + RWLock::RLocker owner_lock(ictx->owner_lock); + if (ictx->exclusive_lock != nullptr) { + ictx->exclusive_lock->block_requests(-EBUSY); + } + } + for (int i = 0; i < snap_count; ++i) { + ImageCtx *ictx = ictxs[i]; + RWLock::RLocker owner_lock(ictx->owner_lock); + + on_finishes[i] = new C_SaferCond; + if (ictx->exclusive_lock != nullptr) { + ictx->exclusive_lock->acquire_lock(on_finishes[i]); + } + } + + ret_code = 0; + for (int i = 0; i < snap_count; ++i) { + r = 0; + ImageCtx *ictx = ictxs[i]; + if (ictx->exclusive_lock != nullptr) { + r = on_finishes[i]->wait(); + } + delete on_finishes[i]; + if (r < 0) { + ret_code = r; + } + } + if (ret_code != 0) { + goto finish; + } + + for (int i = 0; i < snap_count; ++i) { + ImageCtx *ictx = ictxs[i]; + on_finishes[i] = new C_SaferCond; + + RWLock::RLocker owner_locker(ictx->owner_lock); + std::string snap_name; + ictx->snap_lock.get_read(); + snap_t snap_id = get_group_snap_id(ictx, ne); + r = ictx->get_snap_name(snap_id, &snap_name); + ictx->snap_lock.put_read(); + + if (r >= 0) { + ldout(cct, 20) << "rolling back to individual snapshot for image " << ictx->name + << dendl; + ictx->operations->execute_snap_rollback(ne, snap_name, pctx, on_finishes[i]); + } else { + on_finishes[i]->complete(r); + } + } + + for (int i = 0; i < snap_count; ++i) { + r = on_finishes[i]->wait(); + delete on_finishes[i]; + if (r < 0 && r != -ENOENT) { + lderr(cct) << "Failed rolling back group to snapshot. Ret code: " << r << dendl; + ret_code = r; + } + } + +finish: + for (int i = 0; i < snap_count; ++i) { + if (ictxs[i] != nullptr) { + ictxs[i]->state->close(); + } + } + return ret_code; +} + } // anonymous namespace template @@ -1049,6 +1170,45 @@ int Group::snap_list(librados::IoCtx& group_ioctx, const char *group_name, return 0; } +template +int Group::snap_rollback(librados::IoCtx& group_ioctx, + const char *group_name, const char *snap_name, + ProgressContext& pctx) +{ + CephContext *cct = (CephContext *)group_ioctx.cct(); + + string group_id; + int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, + group_name, &group_id); + if (r < 0) { + lderr(cct) << "error reading group id object: " + << cpp_strerror(r) << dendl; + return r; + } + + std::vector snaps; + r = group_snap_list(group_ioctx, group_name, &snaps); + if (r < 0) { + return r; + } + + cls::rbd::GroupSnapshot *group_snap = nullptr; + for (auto &snap : snaps) { + if (snap.name == string(snap_name)) { + group_snap = &snap; + break; + } + } + if (group_snap == nullptr) { + return -ENOENT; + } + + string group_header_oid = util::group_header_name(group_id); + r = group_snap_rollback_by_record(group_ioctx, *group_snap, group_id, + group_header_oid, pctx); + return r; +} + } // namespace api } // namespace librbd diff --git a/src/librbd/api/Group.h b/src/librbd/api/Group.h index 4d3c7612779..6ef55288cf4 100644 --- a/src/librbd/api/Group.h +++ b/src/librbd/api/Group.h @@ -46,6 +46,9 @@ struct Group { const char *old_snap_name, const char *new_snap_name); static int snap_list(librados::IoCtx& group_ioctx, const char *group_name, std::vector *snaps); + static int snap_rollback(librados::IoCtx& group_ioctx, + const char *group_name, const char *snap_name, + ProgressContext& pctx); }; diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index 0d63bfbbec3..14e83b46a3c 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -969,6 +969,32 @@ namespace librbd { return r; } + int RBD::group_snap_rollback(IoCtx& group_ioctx, const char *group_name, + const char *snap_name) { + TracepointProvider::initialize(get_cct(group_ioctx)); + tracepoint(librbd, group_snap_rollback_enter, + group_ioctx.get_pool_name().c_str(), + group_ioctx.get_id(), group_name, snap_name); + librbd::NoOpProgressContext prog_ctx; + int r = librbd::api::Group<>::snap_rollback(group_ioctx, group_name, + snap_name, prog_ctx); + tracepoint(librbd, group_snap_rollback_exit, r); + return r; + } + + int RBD::group_snap_rollback_with_progress(IoCtx& group_ioctx, + const char *group_name, + const char *snap_name, + ProgressContext& prog_ctx) { + TracepointProvider::initialize(get_cct(group_ioctx)); + tracepoint(librbd, group_snap_rollback_enter, + group_ioctx.get_pool_name().c_str(), + group_ioctx.get_id(), group_name, snap_name); + int r = librbd::api::Group<>::snap_rollback(group_ioctx, group_name, + snap_name, prog_ctx); + tracepoint(librbd, group_snap_rollback_exit, r); + return r; + } RBD::AioCompletion::AioCompletion(void *cb_arg, callback_t complete_cb) { @@ -5265,6 +5291,50 @@ extern "C" int rbd_group_snap_list_cleanup(rbd_group_snap_info_t *snaps, return 0; } +extern "C" int rbd_group_snap_rollback(rados_ioctx_t group_p, + const char *group_name, + const char *snap_name) +{ + librados::IoCtx group_ioctx; + librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); + + TracepointProvider::initialize(get_cct(group_ioctx)); + tracepoint(librbd, group_snap_rollback_enter, + group_ioctx.get_pool_name().c_str(), + group_ioctx.get_id(), group_name, snap_name); + + librbd::NoOpProgressContext prog_ctx; + int r = librbd::api::Group<>::snap_rollback(group_ioctx, group_name, + snap_name, prog_ctx); + + tracepoint(librbd, group_snap_rollback_exit, r); + + return r; +} + +extern "C" int rbd_group_snap_rollback_with_progress(rados_ioctx_t group_p, + const char *group_name, + const char *snap_name, + librbd_progress_fn_t cb, + void *cbdata) +{ + librados::IoCtx group_ioctx; + librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); + + TracepointProvider::initialize(get_cct(group_ioctx)); + tracepoint(librbd, group_snap_rollback_enter, + group_ioctx.get_pool_name().c_str(), + group_ioctx.get_id(), group_name, snap_name); + + librbd::CProgressContext prog_ctx(cb, cbdata); + int r = librbd::api::Group<>::snap_rollback(group_ioctx, group_name, + snap_name, prog_ctx); + + tracepoint(librbd, group_snap_rollback_exit, r); + + return r; +} + extern "C" int rbd_snap_get_namespace_type(rbd_image_t image, uint64_t snap_id, rbd_snap_namespace_type_t *namespace_type) { diff --git a/src/test/librbd/test_Groups.cc b/src/test/librbd/test_Groups.cc index 160601758a2..ce3cf2a54ea 100644 --- a/src/test/librbd/test_Groups.cc +++ b/src/test/librbd/test_Groups.cc @@ -207,6 +207,20 @@ TEST_F(TestGroup, add_snapshot) const char *group_name = "snap_group"; const char *snap_name = "snap_snapshot"; + const char orig_data[] = "orig data"; + const char test_data[] = "test data"; + char read_data[10]; + + rbd_image_t image; + ASSERT_EQ(0, rbd_open(ioctx, m_image_name.c_str(), &image, NULL)); + BOOST_SCOPE_EXIT(image) { + EXPECT_EQ(0, rbd_close(image)); + } BOOST_SCOPE_EXIT_END; + + ASSERT_EQ(10, rbd_write(image, 0, 10, orig_data)); + ASSERT_EQ(10, rbd_read(image, 0, 10, read_data)); + ASSERT_EQ(0, memcmp(orig_data, read_data, 10)); + ASSERT_EQ(0, rbd_group_create(ioctx, group_name)); ASSERT_EQ(0, rbd_group_image_add(ioctx, group_name, ioctx, @@ -227,6 +241,14 @@ TEST_F(TestGroup, add_snapshot) ASSERT_STREQ(snap_name, snaps[0].name); + ASSERT_EQ(10, rbd_write(image, 11, 10, test_data)); + ASSERT_EQ(10, rbd_read(image, 11, 10, read_data)); + ASSERT_EQ(0, memcmp(test_data, read_data, 10)); + + ASSERT_EQ(0, rbd_group_snap_rollback(ioctx, group_name, snap_name)); + ASSERT_EQ(10, rbd_read(image, 0, 10, read_data)); + ASSERT_EQ(0, memcmp(orig_data, read_data, 10)); + ASSERT_EQ(0, rbd_group_snap_list_cleanup(snaps, sizeof(rbd_group_snap_info_t), num_snaps)); ASSERT_EQ(0, rbd_group_snap_remove(ioctx, group_name, snap_name)); @@ -255,6 +277,15 @@ TEST_F(TestGroup, add_snapshotPP) ASSERT_EQ(0, rbd.group_image_add(ioctx, group_name, ioctx, m_image_name.c_str())); + librbd::Image image; + ASSERT_EQ(0, rbd.open(ioctx, image, m_image_name.c_str(), NULL)); + bufferlist expect_bl; + bufferlist read_bl; + expect_bl.append(std::string(512, '1')); + ASSERT_EQ((ssize_t)expect_bl.length(), image.write(0, expect_bl.length(), expect_bl)); + ASSERT_EQ(512, image.read(0, 512, read_bl)); + ASSERT_TRUE(expect_bl.contents_equal(read_bl)); + ASSERT_EQ(0, rbd.group_snap_create(ioctx, group_name, snap_name)); std::vector snaps; @@ -264,6 +295,21 @@ TEST_F(TestGroup, add_snapshotPP) ASSERT_EQ(snap_name, snaps[0].name); + bufferlist write_bl; + write_bl.append(std::string(1024, '2')); + ASSERT_EQ(1024, image.write(513, write_bl.length(), write_bl)); + + read_bl.clear(); + ASSERT_EQ(1024, image.read(513, 1024, read_bl)); + ASSERT_TRUE(write_bl.contents_equal(read_bl)); + + ASSERT_EQ(0, rbd.group_snap_rollback(ioctx, group_name, snap_name)); + + ASSERT_EQ(512, image.read(0, 512, read_bl)); + ASSERT_TRUE(expect_bl.contents_equal(read_bl)); + + ASSERT_EQ(0, image.close()); + ASSERT_EQ(0, rbd.group_snap_remove(ioctx, group_name, snap_name)); snaps.clear(); diff --git a/src/tracing/librbd.tp b/src/tracing/librbd.tp index 3c189fb33f0..cd13e45fd9a 100644 --- a/src/tracing/librbd.tp +++ b/src/tracing/librbd.tp @@ -2830,6 +2830,28 @@ TRACEPOINT_EVENT(librbd, group_snap_list_exit, ) ) +TRACEPOINT_EVENT(librbd, group_snap_rollback_enter, + TP_ARGS( + const char*, pool_name, + int64_t, id, + const char*, group_name, + const char*, snap_name), + TP_FIELDS( + ctf_string(pool_name, pool_name) + ctf_integer(int64_t, id, id) + ctf_string(group_name, group_name) + ctf_string(snap_name, snap_name) + ) +) + +TRACEPOINT_EVENT(librbd, group_snap_rollback_exit, + TP_ARGS( + int, retval), + TP_FIELDS( + ctf_integer(int, retval, retval) + ) +) + TRACEPOINT_EVENT(librbd, list_watchers_enter, TP_ARGS( void*, imagectx, -- 2.39.5