From d6f8b92267a78c1b68ec354e7041adff3b880582 Mon Sep 17 00:00:00 2001 From: Victor Denisov Date: Thu, 26 May 2016 20:51:43 -0700 Subject: [PATCH] librbd: Add cg_create, cg_list, cg_remove Signed-off-by: Victor Denisov --- src/include/rbd/librbd.h | 5 + src/include/rbd/librbd.hpp | 5 + src/librbd/Utils.cc | 5 + src/librbd/Utils.h | 2 + src/librbd/internal.cc | 104 +++++++++++++++++++++ src/librbd/internal.h | 5 + src/librbd/librbd.cc | 108 ++++++++++++++++++++++ src/test/Makefile-client.am | 1 + src/test/cli/rbd/help.t | 41 ++++++++ src/test/librbd/test_ConsistencyGroups.cc | 63 +++++++++++++ src/test/librbd/test_main.cc | 2 + src/tracing/librbd.tp | 66 +++++++++++++ 12 files changed, 407 insertions(+) create mode 100644 src/test/librbd/test_ConsistencyGroups.cc diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index fb61b8f70bd3c..f83722b910cc0 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -641,6 +641,11 @@ CEPH_RBD_API int rbd_mirror_image_get_status(rbd_image_t image, rbd_mirror_image_status_t *mirror_image_status, size_t status_size); +// RBD consistency groups support functions +CEPH_RBD_API int rbd_group_create(rados_ioctx_t p, const char *name); +CEPH_RBD_API int rbd_group_remove(rados_ioctx_t p, const char *name); +CEPH_RBD_API int rbd_group_list(rados_ioctx_t p, char *names, size_t *size); + #ifdef __cplusplus } #endif diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp index 4ce59d82d534e..e848fffdf9ae1 100644 --- a/src/include/rbd/librbd.hpp +++ b/src/include/rbd/librbd.hpp @@ -148,6 +148,11 @@ public: int mirror_image_status_summary(IoCtx& io_ctx, std::map *states); + // RBD consistency groups support functions + int group_create(IoCtx& io_ctx, const char *group_name); + int group_remove(IoCtx& io_ctx, const char *group_name); + int group_list(IoCtx& io_ctx, std::vector& names); + private: /* We don't allow assignment or copying */ RBD(const RBD& rhs); diff --git a/src/librbd/Utils.cc b/src/librbd/Utils.cc index 3d73830baa0f3..9cda834ea162d 100644 --- a/src/librbd/Utils.cc +++ b/src/librbd/Utils.cc @@ -8,6 +8,11 @@ namespace librbd { namespace util { +const std::string group_header_name(const std::string &group_id) +{ + return RBD_GROUP_HEADER_PREFIX + group_id; +} + const std::string id_obj_name(const std::string &name) { return RBD_ID_PREFIX + name; diff --git a/src/librbd/Utils.h b/src/librbd/Utils.h index fd881f67f25e1..9ff0b88aa8291 100644 --- a/src/librbd/Utils.h +++ b/src/librbd/Utils.h @@ -5,6 +5,7 @@ #define CEPH_LIBRBD_UTILS_H #include "include/rados/librados.hpp" +#include "include/rbd_types.h" #include "include/Context.h" #include @@ -93,6 +94,7 @@ struct C_AsyncCallback : public Context { } // namespace detail +const std::string group_header_name(const std::string &group_id); const std::string id_obj_name(const std::string &name); const std::string header_name(const std::string &image_id); const std::string old_header_name(const std::string &image_name); diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index 13682df4182ab..4e18a7bb9e4b2 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -3599,4 +3599,108 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force, ictx->perfcounter->inc(l_librbd_readahead_bytes, readahead_length); } } + + // Consistency groups functions + + int group_create(librados::IoCtx& io_ctx, const char *group_name) + { + CephContext *cct = (CephContext *)io_ctx.cct(); + + Rados rados(io_ctx); + uint64_t bid = rados.get_instance_id(); + + uint32_t extra = rand() % 0xFFFFFFFF; + ostringstream bid_ss; + bid_ss << std::hex << bid << std::hex << extra; + string id = bid_ss.str(); + + ldout(cct, 2) << "adding consistency group to directory..." << dendl; + + int r = cls_client::group_dir_add(&io_ctx, RBD_GROUP_DIRECTORY, group_name, id); + if (r < 0) { + lderr(cct) << "error adding consistency group to directory: " + << cpp_strerror(r) + << dendl; + return r; + } + string header_oid = util::group_header_name(id); + + r = cls_client::group_create(&io_ctx, header_oid); + if (r < 0) { + lderr(cct) << "error writing header: " << cpp_strerror(r) << dendl; + goto err_remove_from_dir; + } + + return 0; + + err_remove_from_dir: + int remove_r = cls_client::group_dir_remove(&io_ctx, RBD_GROUP_DIRECTORY, + group_name, id); + if (remove_r < 0) { + lderr(cct) << "error cleaning up consistency group from rbd_directory " + << "object after creation failed: " << cpp_strerror(remove_r) + << dendl; + } + + return r; + } + + int group_remove(librados::IoCtx& io_ctx, const char *group_name) + { + CephContext *cct((CephContext *)io_ctx.cct()); + ldout(cct, 20) << "group_remove " << &io_ctx << " " << group_name << dendl; + + std::string group_id; + int r = cls_client::dir_get_id(&io_ctx, RBD_GROUP_DIRECTORY, + std::string(group_name), &group_id); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "error getting id of group" << dendl; + return r; + } + + string header_oid = util::group_header_name(group_id); + + r = io_ctx.remove(header_oid); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "error removing header: " << cpp_strerror(-r) << dendl; + return r; + } + + r = cls_client::group_dir_remove(&io_ctx, RBD_GROUP_DIRECTORY, + group_name, group_id); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "error removing group from directory" << dendl; + return r; + } + + return 0; + } + + int group_list(IoCtx& io_ctx, vector& names) + { + CephContext *cct = (CephContext *)io_ctx.cct(); + ldout(cct, 20) << "group_list " << &io_ctx << dendl; + + int max_read = 1024; + string last_read = ""; + int r; + do { + map groups; + r = cls_client::group_dir_list(&io_ctx, RBD_GROUP_DIRECTORY, last_read, max_read, &groups); + if (r < 0) { + lderr(cct) << "error listing group in directory: " + << cpp_strerror(r) << dendl; + return r; + } + for (pair group : groups) { + names.push_back(group.first); + } + if (!groups.empty()) { + last_read = groups.rbegin()->first; + } + r = groups.size(); + } while (r == max_read); + + return 0; + } } diff --git a/src/librbd/internal.h b/src/librbd/internal.h index 3fd068691fe0d..bd676be5fd0df 100644 --- a/src/librbd/internal.h +++ b/src/librbd/internal.h @@ -215,6 +215,11 @@ namespace librbd { size_t info_size); int mirror_image_get_status(ImageCtx *ictx, mirror_image_status_t *status, size_t status_size); + + // Consistency groups functions + int group_create(librados::IoCtx& io_ctx, const char *imgname); + int group_remove(librados::IoCtx& io_ctx, const char *group_name); + int group_list(librados::IoCtx& io_ctx, std::vector& names); } #endif diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index 6711b39db1702..982fb11ac498b 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -438,6 +438,42 @@ namespace librbd { return librbd::mirror_image_status_summary(io_ctx, states); } + int RBD::group_create(IoCtx& io_ctx, const char *group_name) + { + TracepointProvider::initialize(get_cct(io_ctx)); + tracepoint(librbd, group_create_enter, io_ctx.get_pool_name().c_str(), + io_ctx.get_id(), group_name); + int r = librbd::group_create(io_ctx, group_name); + tracepoint(librbd, group_create_exit, r); + return r; + } + + int RBD::group_remove(IoCtx& io_ctx, const char *group_name) + { + TracepointProvider::initialize(get_cct(io_ctx)); + tracepoint(librbd, group_remove_enter, io_ctx.get_pool_name().c_str(), + io_ctx.get_id(), group_name); + int r = librbd::group_remove(io_ctx, group_name); + tracepoint(librbd, group_remove_exit, r); + return r; + } + + int RBD::group_list(IoCtx& io_ctx, vector& names) + { + TracepointProvider::initialize(get_cct(io_ctx)); + tracepoint(librbd, group_list_enter, io_ctx.get_pool_name().c_str(), + io_ctx.get_id()); + + int r = librbd::group_list(io_ctx, names); + if (r >= 0) { + for (auto itr : names) { + tracepoint(librbd, group_list_entry, itr.c_str()); + } + } + tracepoint(librbd, group_list_exit, r); + return r; + } + RBD::AioCompletion::AioCompletion(void *cb_arg, callback_t complete_cb) { pc = reinterpret_cast(librbd::AioCompletion::create( @@ -2788,3 +2824,75 @@ extern "C" void rbd_aio_release(rbd_completion_t c) librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c; comp->release(); } + +extern "C" int rbd_group_create(rados_ioctx_t p, const char *name) +{ + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + TracepointProvider::initialize(get_cct(io_ctx)); + tracepoint(librbd, group_create_enter, io_ctx.get_pool_name().c_str(), + io_ctx.get_id(), name); + int r = librbd::group_create(io_ctx, name); + tracepoint(librbd, group_create_exit, r); + return r; +} + +extern "C" int rbd_group_remove(rados_ioctx_t p, const char *name) +{ + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + TracepointProvider::initialize(get_cct(io_ctx)); + tracepoint(librbd, group_remove_enter, io_ctx.get_pool_name().c_str(), + io_ctx.get_id(), name); + int r = librbd::group_remove(io_ctx, name); + tracepoint(librbd, group_remove_exit, r); + return r; +} + +extern "C" int rbd_group_list(rados_ioctx_t p, char *names, size_t *size) +{ + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + TracepointProvider::initialize(get_cct(io_ctx)); + tracepoint(librbd, group_list_enter, io_ctx.get_pool_name().c_str(), + io_ctx.get_id()); + + vector cpp_names; + int r = librbd::list(io_ctx, cpp_names); + + if (r == -ENOENT) { + *size = 0; + *names = '\0'; + tracepoint(librbd, group_list_exit, 0); + return 0; + } + + if (r < 0) { + tracepoint(librbd, group_list_exit, r); + return r; + } + + size_t expected_size = 0; + + for (size_t i = 0; i < cpp_names.size(); i++) { + expected_size += cpp_names[i].size() + 1; + } + if (*size < expected_size) { + *size = expected_size; + tracepoint(librbd, group_list_exit, -ERANGE); + return -ERANGE; + } + + if (!names) + return -EINVAL; + + names[expected_size] = '\0'; + for (int i = 0; i < (int)cpp_names.size(); i++) { + const char* name = cpp_names[i].c_str(); + tracepoint(librbd, group_list_entry, name); + strcpy(names, name); + names += strlen(names) + 1; + } + tracepoint(librbd, group_list_exit, (int)expected_size); + return (int)expected_size; +} diff --git a/src/test/Makefile-client.am b/src/test/Makefile-client.am index 40b7d2df31f4f..0f9af334862ba 100644 --- a/src/test/Makefile-client.am +++ b/src/test/Makefile-client.am @@ -379,6 +379,7 @@ librbd_test_la_SOURCES = \ test/librbd/test_mirroring.cc \ test/librbd/test_MirroringWatcher.cc \ test/librbd/test_ObjectMap.cc \ + test/librbd/test_ConsistencyGroups.cc \ test/librbd/journal/test_Entries.cc \ test/librbd/journal/test_Replay.cc librbd_test_la_CXXFLAGS = $(UNITTEST_CXXFLAGS) diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t index 991b1b761c0b1..2e0cabe74683a 100644 --- a/src/test/cli/rbd/help.t +++ b/src/test/cli/rbd/help.t @@ -20,6 +20,9 @@ feature enable Enable the specified image feature. flatten Fill clone with parent data (make it independent). + group create Create a consistency group. + group list (group ls) List rbd consistency groups. + group remove (group rm) Delete a consistency group. image-meta get Image metadata get the value associated with the key. image-meta list Image metadata list keys with values. @@ -400,6 +403,44 @@ --image arg image name --no-progress disable progress output + rbd help group create + usage: rbd group create [--pool ] [--group ] + + + Create a consistency group. + + Positional arguments + group specification + (example: [/]) + + Optional arguments + -p [ --pool ] arg pool name + --group arg group name + + rbd help group list + usage: rbd group list [--pool ] [--format ] [--pretty-format] + + List rbd consistency groups. + + Optional arguments + -p [ --pool ] arg pool name + --format arg output format [plain, json, or xml] + --pretty-format pretty formatting (json and xml) + + rbd help group remove + usage: rbd group remove [--pool ] [--group ] + + + Delete a consistency group. + + Positional arguments + group specification + (example: [/]) + + Optional arguments + -p [ --pool ] arg pool name + --group arg group name + rbd help image-meta get usage: rbd image-meta get [--pool ] [--image ] diff --git a/src/test/librbd/test_ConsistencyGroups.cc b/src/test/librbd/test_ConsistencyGroups.cc new file mode 100644 index 0000000000000..a6a1c24193c07 --- /dev/null +++ b/src/test/librbd/test_ConsistencyGroups.cc @@ -0,0 +1,63 @@ +// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#include "test/librbd/test_fixture.h" +#include "test/librbd/test_support.h" +#include "include/int_types.h" +#include "include/stringify.h" +#include "include/rados/librados.h" +#include "include/rbd/librbd.hpp" +#include "common/Cond.h" +#include "common/errno.h" +#include "common/Mutex.h" +#include "common/RWLock.h" +#include "cls/lock/cls_lock_client.h" +#include "cls/lock/cls_lock_types.h" +#include "librbd/AioCompletion.h" +#include "librbd/AioImageRequestWQ.h" +#include "librbd/internal.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageWatcher.h" +#include "librbd/WatchNotifyTypes.h" +#include "test/librados/test.h" +#include "gtest/gtest.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ceph; +using namespace boost::assign; +using namespace librbd::watch_notify; + +void register_test_consistency_groups() { +} + +class TestLibCG : public TestFixture { + +}; + +TEST_F(TestLibCG, group_create) +{ + librados::IoCtx ioctx; + ASSERT_EQ(0, _rados.ioctx_create(_pool_name.c_str(), ioctx)); + + librbd::RBD rbd; + ASSERT_EQ(0, rbd.group_create(ioctx, "mygroup")); + + vector groups; + ASSERT_EQ(0, rbd.group_list(ioctx, groups)); + ASSERT_EQ(1U, groups.size()); + ASSERT_EQ("mygroup", groups[0]); + + ASSERT_EQ(0, rbd.group_remove(ioctx, "mygroup")); + + groups.clear(); + ASSERT_EQ(0, rbd.group_list(ioctx, groups)); + ASSERT_EQ(0U, groups.size()); +} diff --git a/src/test/librbd/test_main.cc b/src/test/librbd/test_main.cc index 4ae9f43ee74e7..8338fec0b6cfb 100644 --- a/src/test/librbd/test_main.cc +++ b/src/test/librbd/test_main.cc @@ -17,6 +17,7 @@ extern void register_test_journal_replay(); extern void register_test_object_map(); extern void register_test_mirroring(); extern void register_test_mirroring_watcher(); +extern void register_test_consistency_groups(); #endif // TEST_LIBRBD_INTERNALS int main(int argc, char **argv) @@ -30,6 +31,7 @@ int main(int argc, char **argv) register_test_object_map(); register_test_mirroring(); register_test_mirroring_watcher(); + register_test_consistency_groups(); #endif // TEST_LIBRBD_INTERNALS ::testing::InitGoogleTest(&argc, argv); diff --git a/src/tracing/librbd.tp b/src/tracing/librbd.tp index f91e4e3ee93b3..bbbfae289d3d1 100644 --- a/src/tracing/librbd.tp +++ b/src/tracing/librbd.tp @@ -1795,3 +1795,69 @@ TRACEPOINT_EVENT(librbd, stat_exit, ctf_array_text(char, parent_name, info->parent_name, RBD_MAX_IMAGE_NAME_SIZE) ) ) + +TRACEPOINT_EVENT(librbd, group_create_enter, + TP_ARGS( + const char*, pool_name, + int64_t, id, + const char*, groupname), + TP_FIELDS( + ctf_string(pool_name, pool_name) + ctf_integer(int64_t, id, id) + ctf_string(groupname, groupname) + ) +) + +TRACEPOINT_EVENT(librbd, group_create_exit, + TP_ARGS( + int, retval), + TP_FIELDS( + ctf_integer(int, retval, retval) + ) +) + +TRACEPOINT_EVENT(librbd, group_remove_enter, + TP_ARGS( + const char*, pool_name, + int64_t, id, + const char*, groupname), + TP_FIELDS( + ctf_string(pool_name, pool_name) + ctf_integer(int64_t, id, id) + ctf_string(groupname, groupname) + ) +) + +TRACEPOINT_EVENT(librbd, group_remove_exit, + TP_ARGS( + int, retval), + TP_FIELDS( + ctf_integer(int, retval, retval) + ) +) + +TRACEPOINT_EVENT(librbd, group_list_enter, + TP_ARGS( + const char*, pool_name, + int64_t, id), + TP_FIELDS( + ctf_string(pool_name, pool_name) + ctf_integer(int64_t, id, id) + ) +) + +TRACEPOINT_EVENT(librbd, group_list_entry, + TP_ARGS( + const char*, name), + TP_FIELDS( + ctf_string(name, name) + ) +) + +TRACEPOINT_EVENT(librbd, group_list_exit, + TP_ARGS( + int, retval), + TP_FIELDS( + ctf_integer(int, retval, retval) + ) +) -- 2.39.5