From e53d46487b845c09b8f7c7091e4a885bd1242c29 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 11 Sep 2019 13:16:02 -0400 Subject: [PATCH] librbd: initial support for friendly mirror site names The site name applies to the cluster and stores the friendly name in the MONs config-key store. Signed-off-by: Jason Dillaman (cherry picked from commit a6342665958706c0cd04d45015100aaaae3dc9fa) --- src/include/rados/librados.hpp | 1 + src/include/rbd/librbd.h | 6 + src/include/rbd/librbd.hpp | 5 + src/include/rbd_types.h | 4 +- src/librados/librados_cxx.cc | 10 ++ src/librbd/api/Mirror.cc | 136 ++++++++++++++---- src/librbd/api/Mirror.h | 3 + src/librbd/librbd.cc | 38 +++++ src/pybind/rbd/rbd.pyx | 52 +++++++ .../librados_test_stub/LibradosTestStub.cc | 13 ++ src/test/librbd/test_mirroring.cc | 18 +++ src/test/pybind/test_rbd.py | 6 + 12 files changed, 261 insertions(+), 31 deletions(-) diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp index 539777153c88e..4a0119e985e18 100644 --- a/src/include/rados/librados.hpp +++ b/src/include/rados/librados.hpp @@ -1304,6 +1304,7 @@ inline namespace v14_2_0 { Rados(); explicit Rados(IoCtx& ioctx); ~Rados(); + static void from_rados_t(rados_t cluster, Rados &rados); int init(const char * const id); int init2(const char * const name, const char * const clustername, diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index 1adffe781cf68..8d72a43f80226 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -430,10 +430,16 @@ CEPH_RBD_API void rbd_migration_status_cleanup( rbd_image_migration_status_t *status); /* pool mirroring */ +CEPH_RBD_API int rbd_mirror_site_name_get(rados_t cluster, + char *name, size_t *max_len); +CEPH_RBD_API int rbd_mirror_site_name_set(rados_t cluster, + const char *name); + CEPH_RBD_API int rbd_mirror_mode_get(rados_ioctx_t io_ctx, rbd_mirror_mode_t *mirror_mode); CEPH_RBD_API int rbd_mirror_mode_set(rados_ioctx_t io_ctx, rbd_mirror_mode_t mirror_mode); + CEPH_RBD_API int rbd_mirror_peer_add(rados_ioctx_t io_ctx, char *uuid, size_t uuid_max_length, const char *cluster_name, diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp index 4aaffd116f6f3..d1d2667bd1043 100644 --- a/src/include/rbd/librbd.hpp +++ b/src/include/rbd/librbd.hpp @@ -263,8 +263,13 @@ public: image_migration_status_t *status, size_t status_size); // RBD pool mirroring support functions + int mirror_site_name_get(librados::Rados& rados, std::string* site_name); + int mirror_site_name_set(librados::Rados& rados, + const std::string& site_name); + int mirror_mode_get(IoCtx& io_ctx, rbd_mirror_mode_t *mirror_mode); int mirror_mode_set(IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode); + int mirror_peer_add(IoCtx& io_ctx, std::string *uuid, const std::string &cluster_name, const std::string &client_name); diff --git a/src/include/rbd_types.h b/src/include/rbd_types.h index 94799ab6f1de6..ca42e0b09a043 100644 --- a/src/include/rbd_types.h +++ b/src/include/rbd_types.h @@ -116,7 +116,9 @@ * MON config-key prefix for storing optional remote cluster connectivity * parameters */ -#define RBD_MIRROR_PEER_CONFIG_KEY_PREFIX "rbd/mirror/peer/" +#define RBD_MIRROR_CONFIG_KEY_PREFIX "rbd/mirror/" +#define RBD_MIRROR_SITE_NAME_CONFIG_KEY RBD_MIRROR_CONFIG_KEY_PREFIX "site_name" +#define RBD_MIRROR_PEER_CONFIG_KEY_PREFIX RBD_MIRROR_CONFIG_KEY_PREFIX "peer/" struct rbd_info { __le64 max_id; diff --git a/src/librados/librados_cxx.cc b/src/librados/librados_cxx.cc index 77190788a6785..1069f62d81158 100644 --- a/src/librados/librados_cxx.cc +++ b/src/librados/librados_cxx.cc @@ -2172,6 +2172,16 @@ librados::Rados::~Rados() shutdown(); } +void librados::Rados::from_rados_t(rados_t cluster, Rados &rados) { + if (rados.client) { + rados.client->put(); + } + rados.client = static_cast(cluster); + if (rados.client) { + rados.client->get(); + } +} + int librados::Rados::init(const char * const id) { return rados_create((rados_t *)&client, id); diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index 9fe240bc67097..f20b2a3c6e946 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -22,6 +22,7 @@ #include "librbd/mirror/PromoteRequest.h" #include "librbd/mirror/Types.h" #include "librbd/MirroringWatcher.h" +#include #include #define dout_subsys ceph_subsys_rbd @@ -33,6 +34,56 @@ namespace api { namespace { +int get_config_key(librados::Rados& rados, const std::string& key, + std::string* value) { + std::string cmd = + "{" + "\"prefix\": \"config-key get\", " + "\"key\": \"" + key + "\"" + "}"; + + bufferlist in_bl; + bufferlist out_bl; + + int r = rados.mon_command(cmd, in_bl, &out_bl, nullptr); + if (r == -EINVAL) { + return -EOPNOTSUPP; + } else if (r < 0 && r != -ENOENT) { + return r; + } + + *value = out_bl.to_str(); + return 0; +} + +int set_config_key(librados::Rados& rados, const std::string& key, + const std::string& value) { + std::string cmd; + if (value.empty()) { + cmd = "{" + "\"prefix\": \"config-key rm\", " + "\"key\": \"" + key + "\"" + "}"; + } else { + cmd = "{" + "\"prefix\": \"config-key set\", " + "\"key\": \"" + key + "\", " + "\"val\": \"" + value + "\"" + "}"; + } + bufferlist in_bl; + bufferlist out_bl; + + int r = rados.mon_command(cmd, in_bl, &out_bl, nullptr); + if (r == -EINVAL) { + return -EOPNOTSUPP; + } else if (r < 0) { + return r; + } + + return 0; +} + std::string get_peer_config_key_name(int64_t pool_id, const std::string& peer_uuid) { return RBD_MIRROR_PEER_CONFIG_KEY_PREFIX + stringify(pool_id) + "/" + @@ -42,16 +93,10 @@ std::string get_peer_config_key_name(int64_t pool_id, int remove_peer_config_key(librados::IoCtx& io_ctx, const std::string& peer_uuid) { int64_t pool_id = io_ctx.get_id(); - std::string cmd = - "{" - "\"prefix\": \"config-key rm\", " - "\"key\": \"" + get_peer_config_key_name(pool_id, peer_uuid) + "\"" - "}"; + auto key = get_peer_config_key_name(pool_id, peer_uuid); - bufferlist in_bl; - bufferlist out_bl; librados::Rados rados(io_ctx); - int r = rados.mon_command(cmd, in_bl, &out_bl, nullptr); + int r = set_config_key(rados, key, ""); if (r < 0 && r != -ENOENT && r != -EPERM) { return r; } @@ -531,6 +576,51 @@ int Mirror::image_get_instance_id(I *ictx, std::string *instance_id) { return 0; } +template +int Mirror::site_name_get(librados::Rados& rados, std::string* name) { + CephContext *cct = reinterpret_cast(rados.cct()); + ldout(cct, 20) << dendl; + + int r = get_config_key(rados, RBD_MIRROR_SITE_NAME_CONFIG_KEY, name); + if (r == -EOPNOTSUPP) { + return r; + } else if (r == -ENOENT || name->empty()) { + // default to the cluster fsid + r = rados.cluster_fsid(name); + if (r < 0) { + lderr(cct) << "failed to retrieve cluster fsid: " << cpp_strerror(r) + << dendl; + } + return r; + } else if (r < 0) { + lderr(cct) << "failed to retrieve site name: " << cpp_strerror(r) + << dendl; + return r; + } + + return 0; +} + +template +int Mirror::site_name_set(librados::Rados& rados, const std::string& name) { + CephContext *cct = reinterpret_cast(rados.cct()); + + std::string site_name{name}; + boost::algorithm::trim(site_name); + ldout(cct, 20) << "site_name=" << site_name << dendl; + + int r = set_config_key(rados, RBD_MIRROR_SITE_NAME_CONFIG_KEY, name); + if (r == -EOPNOTSUPP) { + return r; + } else if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to update site name: " << cpp_strerror(r) + << dendl; + return r; + } + + return 0; +} + template int Mirror::mode_get(librados::IoCtx& io_ctx, rbd_mirror_mode_t *mirror_mode) { @@ -890,18 +980,12 @@ int Mirror::peer_get_attributes(librados::IoCtx& io_ctx, ldout(cct, 20) << "uuid=" << uuid << dendl; attributes->clear(); - std::string cmd = - "{" - "\"prefix\": \"config-key get\", " - "\"key\": \"" + get_peer_config_key_name(io_ctx.get_id(), uuid) + "\"" - "}"; - - bufferlist in_bl; - bufferlist out_bl; librados::Rados rados(io_ctx); - int r = rados.mon_command(cmd, in_bl, &out_bl, nullptr); - if (r == -ENOENT || out_bl.length() == 0) { + std::string value; + int r = get_config_key(rados, get_peer_config_key_name(io_ctx.get_id(), uuid), + &value); + if (r == -ENOENT || value.empty()) { return -ENOENT; } else if (r < 0) { lderr(cct) << "failed to retrieve peer attributes: " << cpp_strerror(r) @@ -911,7 +995,7 @@ int Mirror::peer_get_attributes(librados::IoCtx& io_ctx, bool json_valid = false; json_spirit::mValue json_root; - if(json_spirit::read(out_bl.to_str(), json_root)) { + if(json_spirit::read(value, json_root)) { try { auto& json_obj = json_root.get_obj(); for (auto& pairs : json_obj) { @@ -962,18 +1046,10 @@ int Mirror::peer_set_attributes(librados::IoCtx& io_ctx, } ss << "}"; - std::string cmd = - "{" - "\"prefix\": \"config-key set\", " - "\"key\": \"" + get_peer_config_key_name(io_ctx.get_id(), uuid) + "\", " - "\"val\": \"" + ss.str() + "\"" - "}"; - bufferlist in_bl; - bufferlist out_bl; - librados::Rados rados(io_ctx); - r = rados.mon_command(cmd, in_bl, &out_bl, nullptr); - if (r < 0) { + r = set_config_key(rados, get_peer_config_key_name(io_ctx.get_id(), uuid), + ss.str()); + if (r < 0 && r != -ENOENT) { lderr(cct) << "failed to update peer attributes: " << cpp_strerror(r) << dendl; return r; diff --git a/src/librbd/api/Mirror.h b/src/librbd/api/Mirror.h index 78a40849e7fd8..0544a0547c582 100644 --- a/src/librbd/api/Mirror.h +++ b/src/librbd/api/Mirror.h @@ -23,6 +23,9 @@ struct Mirror { typedef std::map IdToMirrorImageStatus; typedef std::map MirrorImageStatusStates; + static int site_name_get(librados::Rados& rados, std::string* name); + static int site_name_set(librados::Rados& rados, const std::string& name); + static int mode_get(librados::IoCtx& io_ctx, rbd_mirror_mode_t *mirror_mode); static int mode_set(librados::IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode); diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index 605fe13323928..39000d57a4d1a 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -840,6 +840,16 @@ namespace librbd { return librbd::api::Mirror<>::mode_get(io_ctx, mirror_mode); } + int RBD::mirror_site_name_get(librados::Rados& rados, + std::string* site_name) { + return librbd::api::Mirror<>::site_name_get(rados, site_name); + } + + int RBD::mirror_site_name_set(librados::Rados& rados, + const std::string& site_name) { + return librbd::api::Mirror<>::site_name_set(rados, site_name); + } + int RBD::mirror_mode_set(IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode) { return librbd::api::Mirror<>::mode_set(io_ctx, mirror_mode); } @@ -2685,6 +2695,34 @@ extern "C" int rbd_image_options_is_empty(rbd_image_options_t opts) } /* pool mirroring */ +extern "C" int rbd_mirror_site_name_get(rados_t cluster, char *name, + size_t *max_len) { + librados::Rados rados; + librados::Rados::from_rados_t(cluster, rados); + + std::string site_name; + int r = librbd::api::Mirror<>::site_name_get(rados, &site_name); + if (r < 0) { + return r; + } + + auto total_len = site_name.size() + 1; + if (*max_len < total_len) { + *max_len = total_len; + return -ERANGE; + } + *max_len = total_len; + + strcpy(name, site_name.c_str()); + return 0; +} + +extern "C" int rbd_mirror_site_name_set(rados_t cluster, const char *name) { + librados::Rados rados; + librados::Rados::from_rados_t(cluster, rados); + return librbd::api::Mirror<>::site_name_set(rados, name); +} + extern "C" int rbd_mirror_mode_get(rados_ioctx_t p, rbd_mirror_mode_t *mirror_mode) { librados::IoCtx io_ctx; diff --git a/src/pybind/rbd/rbd.pyx b/src/pybind/rbd/rbd.pyx index fa9f997492716..6ab0c92bbb690 100644 --- a/src/pybind/rbd/rbd.pyx +++ b/src/pybind/rbd/rbd.pyx @@ -97,6 +97,7 @@ cdef extern from "rbd/librbd.h" nogil: RBD_MAX_BLOCK_NAME_SIZE RBD_MAX_IMAGE_NAME_SIZE + ctypedef void* rados_t ctypedef void* rados_ioctx_t ctypedef void* rbd_image_t ctypedef void* rbd_image_options_t @@ -340,8 +341,12 @@ cdef extern from "rbd/librbd.h" nogil: size_t status_size) void rbd_migration_status_cleanup(rbd_image_migration_status_t *status) + int rbd_mirror_site_name_get(rados_t cluster, char *name, size_t *max_len) + int rbd_mirror_site_name_set(rados_t cluster, const char *name) + int rbd_mirror_mode_get(rados_ioctx_t io, rbd_mirror_mode_t *mirror_mode) int rbd_mirror_mode_set(rados_ioctx_t io, rbd_mirror_mode_t mirror_mode) + int rbd_mirror_peer_add(rados_ioctx_t io, char *uuid, size_t uuid_max_length, const char *cluster_name, const char *client_name) @@ -852,6 +857,9 @@ cdef make_ex(ret, msg, exception_map=errno_to_exception): return OSError(msg, errno=ret) +cdef rados_t convert_rados(rados.Rados rados) except? NULL: + return rados.cluster + cdef rados_ioctx_t convert_ioctx(rados.Ioctx ioctx) except? NULL: return ioctx.io @@ -1644,6 +1652,50 @@ class RBD(object): return status + def mirror_site_name_get(self, rados): + """ + Get the local cluster's friendly site name + + :param rados: cluster connection + :type rados: :class: rados.Rados + :returns: str - local site name + """ + cdef: + rados_t _rados = convert_rados(rados) + char *_site_name = NULL + size_t _max_size = 512 + try: + while True: + _site_name = realloc_chk(_site_name, _max_size) + with nogil: + ret = rbd_mirror_site_name_get(_rados, _site_name, + &_max_size) + if ret >= 0: + break + elif ret != -errno.ERANGE: + raise make_ex(ret, 'error getting site name') + return decode_cstr(_site_name) + finally: + free(_site_name) + + def mirror_site_name_set(self, rados, site_name): + """ + Set the local cluster's friendly site name + + :param rados: cluster connection + :type rados: :class: rados.Rados + :param site_name: friendly site name + :type str: + """ + site_name = cstr(site_name, 'site_name') + cdef: + rados_t _rados = convert_rados(rados) + char *_site_name = site_name + with nogil: + ret = rbd_mirror_site_name_set(_rados, _site_name) + if ret != 0: + raise make_ex(ret, 'error setting mirror site name') + def mirror_mode_get(self, ioctx): """ Get pool mirror mode. diff --git a/src/test/librados_test_stub/LibradosTestStub.cc b/src/test/librados_test_stub/LibradosTestStub.cc index f82d5130d5aeb..66152990cae20 100644 --- a/src/test/librados_test_stub/LibradosTestStub.cc +++ b/src/test/librados_test_stub/LibradosTestStub.cc @@ -990,6 +990,19 @@ Rados::~Rados() { shutdown(); } +void Rados::from_rados_t(rados_t p, Rados &rados) { + if (rados.client != nullptr) { + reinterpret_cast(rados.client)->put(); + rados.client = nullptr; + } + + auto impl = reinterpret_cast(p); + if (impl) { + impl->get(); + rados.client = reinterpret_cast(impl); + } +} + AioCompletion *Rados::aio_create_completion(void *cb_arg, callback_t cb_complete, callback_t cb_safe) { diff --git a/src/test/librbd/test_mirroring.cc b/src/test/librbd/test_mirroring.cc index 1aa72da51774d..f4389a72bcdfb 100644 --- a/src/test/librbd/test_mirroring.cc +++ b/src/test/librbd/test_mirroring.cc @@ -911,3 +911,21 @@ TEST_F(TestMirroring, AioGetStatus) { ASSERT_EQ(0, status.last_update); } } + +TEST_F(TestMirroring, SiteName) { + REQUIRE(!is_librados_test_stub(_rados)); + + const std::string expected_site_name("us-east-1a"); + ASSERT_EQ(0, m_rbd.mirror_site_name_set(_rados, expected_site_name)); + + std::string site_name; + ASSERT_EQ(0, m_rbd.mirror_site_name_get(_rados, &site_name)); + ASSERT_EQ(expected_site_name, site_name); + + ASSERT_EQ(0, m_rbd.mirror_site_name_set(_rados, "")); + + std::string fsid; + ASSERT_EQ(0, _rados.cluster_fsid(&fsid)); + ASSERT_EQ(0, m_rbd.mirror_site_name_get(_rados, &site_name)); + ASSERT_EQ(fsid, site_name); +} diff --git a/src/test/pybind/test_rbd.py b/src/test/pybind/test_rbd.py index 0742ab8069c6f..d1c425dd1623a 100644 --- a/src/test/pybind/test_rbd.py +++ b/src/test/pybind/test_rbd.py @@ -1775,6 +1775,12 @@ class TestMirroring(object): remove_image() self.rbd.mirror_mode_set(ioctx, self.initial_mirror_mode) + def test_site_name(self): + site_name = "us-west-1" + self.rbd.mirror_site_name_set(rados, site_name) + eq(site_name, self.rbd.mirror_site_name_get(rados)) + self.rbd.mirror_site_name_set(rados, "") + eq(rados.get_fsid(), self.rbd.mirror_site_name_get(rados)) def test_mirror_peer(self): eq([], list(self.rbd.mirror_peer_list(ioctx))) -- 2.39.5