From bc82295306e3ffaf0eb6ed94a332fadc04389587 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 14 Nov 2018 21:10:26 +0530 Subject: [PATCH] librbd: new API methods to get/set mirror peer attributes The dashboard will need to manipulate the remote cluster mon_host and key attributes. This logic should be re-used between the rbd CLI and the dashboard. Signed-off-by: Jason Dillaman --- src/include/rbd/librbd.h | 10 ++ src/include/rbd/librbd.hpp | 7 + src/librbd/api/Mirror.cc | 136 +++++++++++++++++- src/librbd/api/Mirror.h | 7 + src/librbd/librbd.cc | 72 ++++++++++ src/pybind/rbd/rbd.pyx | 85 +++++++++++ .../librados_test_stub/TestRadosClient.cc | 2 + src/test/librbd/test_librbd.cc | 34 +++++ src/test/pybind/test_rbd.py | 12 +- 9 files changed, 363 insertions(+), 2 deletions(-) diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index 8a4196cf6a1..cbbd42b90e1 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -119,6 +119,9 @@ typedef struct { char *client_name; } rbd_mirror_peer_t; +#define RBD_MIRROR_PEER_ATTRIBUTE_NAME_MON_HOST "mon_host" +#define RBD_MIRROR_PEER_ATTRIBUTE_NAME_KEY "key" + typedef enum { RBD_MIRROR_IMAGE_DISABLING = 0, RBD_MIRROR_IMAGE_ENABLED = 1, @@ -407,6 +410,13 @@ CEPH_RBD_API int rbd_mirror_peer_set_client(rados_ioctx_t io_ctx, CEPH_RBD_API int rbd_mirror_peer_set_cluster(rados_ioctx_t io_ctx, const char *uuid, const char *cluster_name); +CEPH_RBD_API int rbd_mirror_peer_get_attributes( + rados_ioctx_t p, const char *uuid, char *keys, size_t *max_key_len, + char *values, size_t *max_value_len, size_t *key_value_count); +CEPH_RBD_API int rbd_mirror_peer_set_attributes( + rados_ioctx_t p, const char *uuid, const char *keys, const char *values, + size_t key_value_count); + CEPH_RBD_API int rbd_mirror_image_status_list(rados_ioctx_t io_ctx, const char *start_id, size_t max, char **image_ids, diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp index 3b8d2cdffe8..93703b1b40a 100644 --- a/src/include/rbd/librbd.hpp +++ b/src/include/rbd/librbd.hpp @@ -247,6 +247,13 @@ public: const std::string &client_name); int mirror_peer_set_cluster(IoCtx& io_ctx, const std::string &uuid, const std::string &cluster_name); + int mirror_peer_get_attributes( + IoCtx& io_ctx, const std::string &uuid, + std::map *key_vals); + int mirror_peer_set_attributes( + IoCtx& io_ctx, const std::string &uuid, + const std::map& key_vals); + int mirror_image_status_list(IoCtx& io_ctx, const std::string &start_id, size_t max, std::map *images); int mirror_image_status_summary(IoCtx& io_ctx, diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index 9e3619b1223..f02503828c7 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -3,6 +3,8 @@ #include "librbd/api/Mirror.h" #include "include/rados/librados.hpp" +#include "include/stringify.h" +#include "common/ceph_json.h" #include "common/dout.h" #include "common/errno.h" #include "cls/rbd/cls_rbd_client.h" @@ -31,6 +33,31 @@ namespace api { namespace { +std::string get_peer_config_key_name(int64_t pool_id, + const std::string& peer_uuid) { + return RBD_MIRROR_PEER_CONFIG_KEY_PREFIX + stringify(pool_id) + "/" + + peer_uuid; +} + +int remove_peer_config_key(librados::IoCtx& io_ctx, + const std::string& peer_uuid) { + int64_t pool_id = io_ctx.get_id(); + std::string cmd = + "{" + "\"prefix\": \"config-key rm\", " + "\"key\": \"" + get_peer_config_key_name(pool_id, peer_uuid) + "\"" + "}"; + + bufferlist in_bl; + bufferlist out_bl; + librados::Rados rados(io_ctx); + int r = rados.mon_command(cmd, in_bl, &out_bl, nullptr); + if (r < 0 && r != -ENOENT && r != -EPERM) { + return r; + } + return 0; +} + template int validate_mirroring_enabled(I *ictx) { CephContext *cct = ictx->cct; @@ -768,7 +795,14 @@ int Mirror::peer_remove(librados::IoCtx& io_ctx, const std::string &uuid) { CephContext *cct = reinterpret_cast(io_ctx.cct()); ldout(cct, 20) << "uuid=" << uuid << dendl; - int r = cls_client::mirror_peer_remove(&io_ctx, uuid); + int r = remove_peer_config_key(io_ctx, uuid); + if (r < 0) { + lderr(cct) << "failed to remove peer attributes '" << uuid << "': " + << cpp_strerror(r) << dendl; + return r; + } + + r = cls_client::mirror_peer_remove(&io_ctx, uuid); if (r < 0 && r != -ENOENT) { lderr(cct) << "failed to remove peer '" << uuid << "': " << cpp_strerror(r) << dendl; @@ -840,6 +874,106 @@ int Mirror::peer_set_cluster(librados::IoCtx& io_ctx, return 0; } +template +int Mirror::peer_get_attributes(librados::IoCtx& io_ctx, + const std::string &uuid, + Attributes* attributes) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << "uuid=" << uuid << dendl; + + attributes->clear(); + std::string cmd = + "{" + "\"prefix\": \"config-key get\", " + "\"key\": \"" + get_peer_config_key_name(io_ctx.get_id(), uuid) + "\"" + "}"; + + bufferlist in_bl; + bufferlist out_bl; + + librados::Rados rados(io_ctx); + int r = rados.mon_command(cmd, in_bl, &out_bl, nullptr); + if (r == -ENOENT || out_bl.length() == 0) { + return -ENOENT; + } else if (r < 0) { + lderr(cct) << "failed to retrieve peer attributes: " << cpp_strerror(r) + << dendl; + return r; + } + + bool json_valid = false; + json_spirit::mValue json_root; + if(json_spirit::read(out_bl.to_str(), json_root)) { + try { + auto& json_obj = json_root.get_obj(); + for (auto& pairs : json_obj) { + (*attributes)[pairs.first] = pairs.second.get_str(); + } + json_valid = true; + } catch (std::runtime_error&) { + } + } + + if (!json_valid) { + lderr(cct) << "invalid peer attributes JSON received" << dendl; + return -EINVAL; + } + return 0; +} + +template +int Mirror::peer_set_attributes(librados::IoCtx& io_ctx, + const std::string &uuid, + const Attributes& attributes) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << "uuid=" << uuid << ", " + << "attributes=" << attributes << dendl; + + std::vector mirror_peers; + int r = peer_list(io_ctx, &mirror_peers); + if (r < 0) { + return r; + } + + if (std::find_if(mirror_peers.begin(), mirror_peers.end(), + [&uuid](const librbd::mirror_peer_t& peer) { + return uuid == peer.uuid; + }) == mirror_peers.end()) { + ldout(cct, 5) << "mirror peer uuid " << uuid << " does not exist" << dendl; + return -ENOENT; + } + + std::stringstream ss; + ss << "{"; + for (auto& pair : attributes) { + ss << "\\\"" << pair.first << "\\\": " + << "\\\"" << pair.second << "\\\""; + if (&pair != &(*attributes.rbegin())) { + ss << ", "; + } + } + ss << "}"; + + std::string cmd = + "{" + "\"prefix\": \"config-key set\", " + "\"key\": \"" + get_peer_config_key_name(io_ctx.get_id(), uuid) + "\", " + "\"val\": \"" + ss.str() + "\"" + "}"; + bufferlist in_bl; + bufferlist out_bl; + + librados::Rados rados(io_ctx); + r = rados.mon_command(cmd, in_bl, &out_bl, nullptr); + if (r < 0) { + lderr(cct) << "failed to update peer attributes: " << cpp_strerror(r) + << dendl; + return r; + } + + return 0; +} + template int Mirror::image_status_list(librados::IoCtx& io_ctx, const std::string &start_id, size_t max, diff --git a/src/librbd/api/Mirror.h b/src/librbd/api/Mirror.h index c2bc4464453..78a40849e7f 100644 --- a/src/librbd/api/Mirror.h +++ b/src/librbd/api/Mirror.h @@ -19,6 +19,7 @@ namespace api { template struct Mirror { + typedef std::map Attributes; typedef std::map IdToMirrorImageStatus; typedef std::map MirrorImageStatusStates; @@ -35,6 +36,12 @@ struct Mirror { const std::string &client_name); static int peer_set_cluster(librados::IoCtx& io_ctx, const std::string &uuid, const std::string &cluster_name); + static int peer_get_attributes(librados::IoCtx& io_ctx, + const std::string &uuid, + Attributes* attributes); + static int peer_set_attributes(librados::IoCtx& io_ctx, + const std::string &uuid, + const Attributes& attributes); static int image_status_list(librados::IoCtx& io_ctx, const std::string &start_id, size_t max, diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index bd6c9e9e2b5..5f9aacd191d 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -831,6 +831,18 @@ namespace librbd { return librbd::api::Mirror<>::peer_set_cluster(io_ctx, uuid, cluster_name); } + int RBD::mirror_peer_get_attributes( + IoCtx& io_ctx, const std::string &uuid, + std::map *key_vals) { + return librbd::api::Mirror<>::peer_get_attributes(io_ctx, uuid, key_vals); + } + + int RBD::mirror_peer_set_attributes( + IoCtx& io_ctx, const std::string &uuid, + const std::map& key_vals) { + return librbd::api::Mirror<>::peer_set_attributes(io_ctx, uuid, key_vals); + } + int RBD::mirror_image_status_list(IoCtx& io_ctx, const std::string &start_id, size_t max, std::map *images) { return librbd::api::Mirror<>::image_status_list(io_ctx, start_id, max, @@ -2638,6 +2650,66 @@ extern "C" int rbd_mirror_peer_set_cluster(rados_ioctx_t p, const char *uuid, return librbd::api::Mirror<>::peer_set_cluster(io_ctx, uuid, cluster_name); } +extern "C" int rbd_mirror_peer_get_attributes( + rados_ioctx_t p, const char *uuid, char *keys, size_t *max_key_len, + char *values, size_t *max_val_len, size_t *key_value_count) { + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + + std::map attributes; + int r = librbd::api::Mirror<>::peer_get_attributes(io_ctx, uuid, &attributes); + if (r < 0) { + return r; + } + + size_t key_total_len = 0, val_total_len = 0; + for (auto& it : attributes) { + key_total_len += it.first.size() + 1; + val_total_len += it.second.length() + 1; + } + + bool too_short = ((*max_key_len < key_total_len) || + (*max_val_len < val_total_len)); + + *max_key_len = key_total_len; + *max_val_len = val_total_len; + *key_value_count = attributes.size(); + if (too_short) { + return -ERANGE; + } + + char *keys_p = keys; + char *values_p = values; + for (auto& it : attributes) { + strncpy(keys_p, it.first.c_str(), it.first.size() + 1); + keys_p += it.first.size() + 1; + + strncpy(values_p, it.second.c_str(), it.second.length() + 1); + values_p += it.second.length() + 1; + } + + return 0; +} + +extern "C" int rbd_mirror_peer_set_attributes( + rados_ioctx_t p, const char *uuid, const char *keys, const char *values, + size_t count) { + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + + std::map attributes; + + for (size_t i = 0; i < count; ++i) { + const char* key = keys; + keys += strlen(key) + 1; + const char* value = values; + values += strlen(value) + 1; + attributes[key] = value; + } + + return librbd::api::Mirror<>::peer_set_attributes(io_ctx, uuid, attributes); +} + extern "C" int rbd_mirror_image_status_list(rados_ioctx_t p, const char *start_id, size_t max, char **image_ids, rbd_mirror_image_status_t *images, size_t *len) { diff --git a/src/pybind/rbd/rbd.pyx b/src/pybind/rbd/rbd.pyx index 2b2b909d440..a1b58888478 100644 --- a/src/pybind/rbd/rbd.pyx +++ b/src/pybind/rbd/rbd.pyx @@ -25,6 +25,7 @@ from libc.string cimport strdup from collections import Iterable from datetime import datetime +from itertools import chain cimport rados @@ -129,6 +130,9 @@ cdef extern from "rbd/librbd.h" nogil: char *cluster_name char *client_name + cdef char* _RBD_MIRROR_PEER_ATTRIBUTE_NAME_MON_HOST "RBD_MIRROR_PEER_ATTRIBUTE_NAME_MON_HOST" + cdef char* _RBD_MIRROR_PEER_ATTRIBUTE_NAME_KEY "RBD_MIRROR_PEER_ATTRIBUTE_NAME_KEY" + ctypedef enum rbd_mirror_image_state_t: _RBD_MIRROR_IMAGE_DISABLING "RBD_MIRROR_IMAGE_DISABLING" _RBD_MIRROR_IMAGE_ENABLED "RBD_MIRROR_IMAGE_ENABLED" @@ -305,6 +309,14 @@ cdef extern from "rbd/librbd.h" nogil: const char *client_name) int rbd_mirror_peer_set_cluster(rados_ioctx_t io_ctx, const char *uuid, const char *cluster_name) + int rbd_mirror_peer_get_attributes(rados_ioctx_t io_ctx, const char *uuid, + char *keys, size_t *max_key_len, + char *values, size_t *max_val_length, + size_t *key_value_count) + int rbd_mirror_peer_set_attributes(rados_ioctx_t io_ctx, const char *uuid, + const char *keys, const char *values, + size_t count) + int rbd_mirror_image_status_list(rados_ioctx_t io, const char *start_id, size_t max, char **image_ids, rbd_mirror_image_status_t *images, @@ -583,6 +595,9 @@ RBD_MIRROR_MODE_DISABLED = _RBD_MIRROR_MODE_DISABLED RBD_MIRROR_MODE_IMAGE = _RBD_MIRROR_MODE_IMAGE RBD_MIRROR_MODE_POOL = _RBD_MIRROR_MODE_POOL +RBD_MIRROR_PEER_ATTRIBUTE_NAME_MON_HOST = _RBD_MIRROR_PEER_ATTRIBUTE_NAME_MON_HOST +RBD_MIRROR_PEER_ATTRIBUTE_NAME_KEY = _RBD_MIRROR_PEER_ATTRIBUTE_NAME_KEY + RBD_MIRROR_IMAGE_DISABLING = _RBD_MIRROR_IMAGE_DISABLING RBD_MIRROR_IMAGE_ENABLED = _RBD_MIRROR_IMAGE_ENABLED RBD_MIRROR_IMAGE_DISABLED = _RBD_MIRROR_IMAGE_DISABLED @@ -1610,6 +1625,76 @@ class RBD(object): if ret != 0: raise make_ex(ret, 'error setting mirror peer cluster') + def mirror_peer_get_attributes(self, ioctx, uuid): + """ + Get optional mirror peer attributes + + :param ioctx: determines which RADOS pool is written + :type ioctx: :class:`rados.Ioctx` + :param uuid: uuid of the mirror peer + :type uuid: str + + :returns: dict - contains the following keys: + + * ``mon_host`` (str) - monitor addresses + + * ``key`` (str) - CephX key + """ + uuid = cstr(uuid, 'uuid') + cdef: + rados_ioctx_t _ioctx = convert_ioctx(ioctx) + char *_uuid = uuid + char *_keys = NULL + char *_vals = NULL + size_t _keys_size = 512 + size_t _vals_size = 512 + size_t _count = 0 + try: + while True: + _keys = realloc_chk(_keys, _keys_size) + _vals = realloc_chk(_vals, _vals_size) + with nogil: + ret = rbd_mirror_peer_get_attributes(_ioctx, _uuid, _keys, + &_keys_size, _vals, + &_vals_size, &_count) + if ret >= 0: + break + elif ret != -errno.ERANGE: + raise make_ex(ret, 'error getting mirror peer attributes') + keys = [decode_cstr(x) for x in _keys[:_keys_size].split(b'\0') if x] + vals = [decode_cstr(x) for x in _vals[:_vals_size].split(b'\0') if x] + return dict(zip(keys, vals)) + finally: + free(_keys) + free(_vals) + + def mirror_peer_set_attributes(self, ioctx, uuid, attributes): + """ + Set optional mirror peer attributes + + :param ioctx: determines which RADOS pool is written + :type ioctx: :class:`rados.Ioctx` + :param uuid: uuid of the mirror peer + :type uuid: str + :param attributes: 'mon_host' and 'key' attributes + :type attributes: dict + """ + uuid = cstr(uuid, 'uuid') + keys_str = '\0'.join([cstr(x[0], 'key') for x in attributes.items()]) + vals_str = '\0'.join([cstr(x[1], 'val') for x in attributes.items()]) + cdef: + rados_ioctx_t _ioctx = convert_ioctx(ioctx) + char *_uuid = uuid + char *_keys = keys_str + char *_vals = vals_str + size_t _count = len(attributes) + + with nogil: + ret = rbd_mirror_peer_set_attributes(_ioctx, _uuid, _keys, _vals, + _count) + if ret != 0: + raise make_ex(ret, 'error setting mirror peer attributes') + def mirror_image_status_list(self, ioctx): """ Iterate over the mirror image statuses of a pool. diff --git a/src/test/librados_test_stub/TestRadosClient.cc b/src/test/librados_test_stub/TestRadosClient.cc index a5642460cc8..65c011fbda1 100644 --- a/src/test/librados_test_stub/TestRadosClient.cc +++ b/src/test/librados_test_stub/TestRadosClient.cc @@ -166,6 +166,8 @@ int TestRadosClient::mon_command(const std::vector& cmd, return 0; } else if ((*j_it)->get_data() == "osd tier remove") { return 0; + } else if ((*j_it)->get_data() == "config-key rm") { + return 0; } } return -ENOSYS; diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc index fd3373d4de4..9a6a8af842b 100644 --- a/src/test/librbd/test_librbd.cc +++ b/src/test/librbd/test_librbd.cc @@ -6231,6 +6231,40 @@ TEST_F(TestLibRBD, Mirror) { ASSERT_EQ(expected_peers, peers); ASSERT_EQ(-EBUSY, rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_DISABLED)); + ASSERT_EQ(0, rbd.mirror_peer_remove(ioctx, uuid1)); + ASSERT_EQ(0, rbd.mirror_peer_remove(ioctx, uuid3)); + ASSERT_EQ(0, rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_DISABLED)); +} + +TEST_F(TestLibRBD, MirrorPeerAttributes) { + REQUIRE(!is_librados_test_stub(_rados)); + + librados::IoCtx ioctx; + ASSERT_EQ(0, _rados.ioctx_create(m_pool_name.c_str(), ioctx)); + + librbd::RBD rbd; + ASSERT_EQ(0, rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_IMAGE)); + + std::string uuid; + ASSERT_EQ(0, rbd.mirror_peer_add(ioctx, &uuid, "remote_cluster", "client")); + + std::map attributes; + ASSERT_EQ(-ENOENT, rbd.mirror_peer_get_attributes(ioctx, uuid, &attributes)); + ASSERT_EQ(-ENOENT, rbd.mirror_peer_set_attributes(ioctx, "missing uuid", + attributes)); + + std::map expected_attributes{ + {"mon_host", "1.2.3.4"}, + {"key", "ABC"}}; + ASSERT_EQ(0, rbd.mirror_peer_set_attributes(ioctx, uuid, + expected_attributes)); + + ASSERT_EQ(0, rbd.mirror_peer_get_attributes(ioctx, uuid, + &attributes)); + ASSERT_EQ(expected_attributes, attributes); + + ASSERT_EQ(0, rbd.mirror_peer_remove(ioctx, uuid)); + ASSERT_EQ(0, rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_DISABLED)); } TEST_F(TestLibRBD, FlushCacheWithCopyupOnExternalSnapshot) { diff --git a/src/test/pybind/test_rbd.py b/src/test/pybind/test_rbd.py index f9f5131ccd7..ac09c99db13 100644 --- a/src/test/pybind/test_rbd.py +++ b/src/test/pybind/test_rbd.py @@ -24,7 +24,9 @@ from rbd import (RBD, Group, Image, ImageNotFound, InvalidArgument, ImageExists, RBD_LOCK_MODE_EXCLUSIVE, RBD_OPERATION_FEATURE_GROUP, RBD_SNAP_NAMESPACE_TYPE_TRASH, RBD_IMAGE_MIGRATION_STATE_PREPARED, RBD_CONFIG_SOURCE_CONFIG, - RBD_CONFIG_SOURCE_POOL, RBD_CONFIG_SOURCE_IMAGE) + RBD_CONFIG_SOURCE_POOL, RBD_CONFIG_SOURCE_IMAGE, + RBD_MIRROR_PEER_ATTRIBUTE_NAME_MON_HOST, + RBD_MIRROR_PEER_ATTRIBUTE_NAME_KEY) rados = None ioctx = None @@ -1717,6 +1719,14 @@ class TestMirroring(object): 'client_name' : client_name, } eq([peer], list(self.rbd.mirror_peer_list(ioctx))) + + attribs = { + RBD_MIRROR_PEER_ATTRIBUTE_NAME_MON_HOST: 'host1', + RBD_MIRROR_PEER_ATTRIBUTE_NAME_KEY: 'abc' + } + self.rbd.mirror_peer_set_attributes(ioctx, uuid, attribs) + eq(attribs, self.rbd.mirror_peer_get_attributes(ioctx, uuid)) + self.rbd.mirror_peer_remove(ioctx, uuid) eq([], list(self.rbd.mirror_peer_list(ioctx))) -- 2.39.5