From: Jason Dillaman Date: Wed, 23 Sep 2015 22:10:25 +0000 (-0400) Subject: cls_rbd: new methods for handling mirroring X-Git-Tag: v10.0.2~113^2~6 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=54b1c8dea28a118cda329ca7a5e7b14c73655022;p=ceph.git cls_rbd: new methods for handling mirroring Signed-off-by: Jason Dillaman --- diff --git a/src/cls/CMakeLists.txt b/src/cls/CMakeLists.txt index 36b2a5b82897..9cc3a70391c3 100644 --- a/src/cls/CMakeLists.txt +++ b/src/cls/CMakeLists.txt @@ -7,11 +7,11 @@ install(TARGETS cls_hello DESTINATION lib/rados-classes) # cls_rbd if (WITH_RBD) - add_library(cls_rbd SHARED rbd/cls_rbd.cc) + add_library(cls_rbd SHARED rbd/cls_rbd.cc rbd/cls_rbd_types.cc) set_target_properties(cls_rbd PROPERTIES VERSION "1.0.0" SOVERSION "1") install(TARGETS cls_rbd DESTINATION lib/rados-classes) - add_library(cls_rbd_client rbd/cls_rbd_client.cc) + add_library(cls_rbd_client rbd/cls_rbd_client.cc rbd/cls_rbd_types.cc) endif (WITH_RBD) # cls_lock diff --git a/src/cls/Makefile-client.am b/src/cls/Makefile-client.am index f1f798330606..dcecc2802d3b 100644 --- a/src/cls/Makefile-client.am +++ b/src/cls/Makefile-client.am @@ -41,7 +41,9 @@ libcls_rgw_client_la_SOURCES = \ noinst_LTLIBRARIES += libcls_rgw_client.la DENCODER_DEPS += libcls_rgw_client.la -libcls_rbd_client_la_SOURCES = cls/rbd/cls_rbd_client.cc +libcls_rbd_client_la_SOURCES = \ + cls/rbd/cls_rbd_client.cc \ + cls/rbd/cls_rbd_types.cc noinst_LTLIBRARIES += libcls_rbd_client.la libcls_user_client_a_SOURCES = cls/user/cls_user_client.cc \ @@ -71,6 +73,7 @@ noinst_HEADERS += \ cls/numops/cls_numops_client.h \ cls/rbd/cls_rbd.h \ cls/rbd/cls_rbd_client.h \ + cls/rbd/cls_rbd_types.h \ cls/refcount/cls_refcount_ops.h \ cls/refcount/cls_refcount_client.h \ cls/version/cls_version_types.h \ diff --git a/src/cls/Makefile-server.am b/src/cls/Makefile-server.am index cf0b26a21c66..9b081bc97189 100644 --- a/src/cls/Makefile-server.am +++ b/src/cls/Makefile-server.am @@ -10,7 +10,9 @@ libcls_numops_la_SOURCES = cls/numops/cls_numops.cc libcls_numops_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' radoslib_LTLIBRARIES += libcls_numops.la -libcls_rbd_la_SOURCES = cls/rbd/cls_rbd.cc +libcls_rbd_la_SOURCES = \ + cls/rbd/cls_rbd.cc \ + cls/rbd/cls_rbd_types.cc libcls_rbd_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) libcls_rbd_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' radoslib_LTLIBRARIES += libcls_rbd.la diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc index ab40c16b7330..d1e7caa4c5c5 100644 --- a/src/cls/rbd/cls_rbd.cc +++ b/src/cls/rbd/cls_rbd.cc @@ -44,6 +44,7 @@ #include "include/rbd/object_map_types.h" #include "cls/rbd/cls_rbd.h" +#include "cls/rbd/cls_rbd_types.h" /* @@ -110,6 +111,13 @@ cls_method_handle_t h_old_snapshots_list; cls_method_handle_t h_old_snapshot_add; cls_method_handle_t h_old_snapshot_remove; cls_method_handle_t h_old_snapshot_rename; +cls_method_handle_t h_mirror_is_enabled; +cls_method_handle_t h_mirror_set_enabled; +cls_method_handle_t h_mirror_peer_list; +cls_method_handle_t h_mirror_peer_add; +cls_method_handle_t h_mirror_peer_remove; +cls_method_handle_t h_mirror_peer_set_client; +cls_method_handle_t h_mirror_peer_set_cluster; #define RBD_MAX_KEYS_READ 64 #define RBD_SNAP_KEY_PREFIX "snapshot_" @@ -2901,7 +2909,339 @@ int old_snapshot_rename(cls_method_context_t hctx, bufferlist *in, bufferlist *o rc = cls_cxx_write_full(hctx, &newbl); if (rc < 0) return rc; + return 0; +} + +namespace mirror { + +static const std::string PEER_KEY_PREFIX("mirror_peer_"); + +std::string peer_key(const std::string &uuid) { + return PEER_KEY_PREFIX + uuid; +} + +int is_enabled(cls_method_context_t hctx, bool *enabled) { + bufferlist bl; + int r = cls_cxx_map_get_val(hctx, "mirror_enabled", &bl); + if (r < 0 && r != -ENOENT) { + CLS_ERR("error reading mirror enabled flag: %s", + cpp_strerror(r).c_str()); + return r; + } + + if (r == 0) { + try { + bufferlist::iterator bl_it = bl.begin(); + ::decode(*enabled, bl_it); + } catch (const buffer::error &err) { + CLS_ERR("could not decode flag"); + return -EIO; + } + } else { + *enabled = false; + } + return 0; +} + +int read_peers(cls_method_context_t hctx, + std::vector *peers) { + std::string last_read = PEER_KEY_PREFIX; + int max_read = RBD_MAX_KEYS_READ; + int r = max_read; + while (r == max_read) { + std::map vals; + r = cls_cxx_map_get_vals(hctx, last_read, PEER_KEY_PREFIX.c_str(), + max_read, &vals); + if (r < 0) { + CLS_ERR("error reading peers: %s", cpp_strerror(r).c_str()); + return r; + } + + for (auto &it : vals) { + try { + bufferlist::iterator bl_it = it.second.begin(); + cls::rbd::MirrorPeer peer; + ::decode(peer, bl_it); + peers->push_back(peer); + } catch (const buffer::error &err) { + CLS_ERR("could not decode peer '%s'", it.first.c_str()); + return -EIO; + } + } + } + return 0; +} + +int read_peer(cls_method_context_t hctx, const std::string uuid, + cls::rbd::MirrorPeer *peer) { + bufferlist bl; + int r = cls_cxx_map_get_val(hctx, peer_key(uuid), &bl); + if (r < 0) { + CLS_ERR("error reading peer '%s': %s", uuid.c_str(), + cpp_strerror(r).c_str()); + return r; + } + + try { + bufferlist::iterator bl_it = bl.begin(); + ::decode(*peer, bl_it); + } catch (const buffer::error &err) { + CLS_ERR("could not decode peer '%s'", uuid.c_str()); + return -EIO; + } + return 0; +} + +int write_peer(cls_method_context_t hctx, const std::string uuid, + const cls::rbd::MirrorPeer &peer) { + bufferlist bl; + ::encode(peer, bl); + + int r = cls_cxx_map_set_val(hctx, peer_key(uuid), &bl); + if (r < 0) { + CLS_ERR("error writing peer '%s': %s", uuid.c_str(), + cpp_strerror(r).c_str()); + return r; + } + return 0; +} + +} // namespace mirror + +/** + * Input: + * none + * + * Output: + * @param bool: true if enabled + * @returns 0 on success, negative error code on failure + */ +int mirror_is_enabled(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + bool enabled; + int r = mirror::is_enabled(hctx, &enabled); + if (r < 0) { + return r; + } + + ::encode(enabled, *out); + return 0; +} + +/** + * Input: + * @param enabled (bool) + * + * Output: + * @returns 0 on success, negative error code on failure + */ +int mirror_set_enabled(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + bool enabled; + try { + bufferlist::iterator bl_it = in->begin(); + ::decode(enabled, bl_it); + } catch (const buffer::error &err) { + return -EINVAL; + } + + int r; + if (enabled) { + bufferlist bl; + ::encode(enabled, bl); + + r = cls_cxx_map_set_val(hctx, "mirror_enabled", &bl); + if (r < 0) { + CLS_ERR("error enabling mirroring: %s", cpp_strerror(r).c_str()); + return r; + } + } else { + std::vector peers; + int r = mirror::read_peers(hctx, &peers); + if (r < 0 && r != -ENOENT) { + return r; + } + + if (!peers.empty()) { + CLS_ERR("mirroring peers still registered"); + return -EBUSY; + } + + r = cls_cxx_map_remove_key(hctx, "mirror_enabled"); + if (r < 0 && r != -ENOENT) { + CLS_ERR("error disabling mirroring: %s", cpp_strerror(r).c_str()); + return r; + } + } + return 0; +} + +/** + * Input: + * none + * + * Output: + * @param std::vector: collection of peers + * @returns 0 on success, negative error code on failure + */ +int mirror_peer_list(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + std::vector peers; + int r = mirror::read_peers(hctx, &peers); + if (r < 0 && r != -ENOENT) { + return r; + } + + ::encode(peers, *out); + return 0; +} + +/** + * Input: + * @param mirror_peer (cls::rbd::MirrorPeer) + * + * Output: + * @returns 0 on success, negative error code on failure + */ +int mirror_peer_add(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + cls::rbd::MirrorPeer mirror_peer; + try { + bufferlist::iterator it = in->begin(); + ::decode(mirror_peer, it); + } catch (const buffer::error &err) { + return -EINVAL; + } + + bool enabled; + int r = mirror::is_enabled(hctx, &enabled); + if (r < 0) { + return r; + } + if (!enabled) { + CLS_ERR("mirroring must be enabled on the pool"); + return -EINVAL; + } + + std::vector peers; + r = mirror::read_peers(hctx, &peers); + if (r < 0 && r != -ENOENT) { + return r; + } + + for (auto const &peer : peers) { + if (peer.cluster_uuid == mirror_peer.cluster_uuid) { + CLS_ERR("peer cluster uuid '%s' alread exists", + peer.cluster_uuid.c_str()); + return -EEXIST; + } else if (peer.cluster_name == mirror_peer.cluster_name) { + CLS_ERR("peer cluster name '%s' alread exists", + peer.cluster_name.c_str()); + return -EEXIST; + } + } + + bufferlist bl; + ::encode(mirror_peer, bl); + r = cls_cxx_map_set_val(hctx, mirror::peer_key(mirror_peer.cluster_uuid), + &bl); + if (r < 0) { + CLS_ERR("error adding peer: %s", cpp_strerror(r).c_str()); + return r; + } + return 0; +} + +/** + * Input: + * @param cluster_uuid (std::string) + * + * Output: + * @returns 0 on success, negative error code on failure + */ +int mirror_peer_remove(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + std::string cluster_uuid; + try { + bufferlist::iterator it = in->begin(); + ::decode(cluster_uuid, it); + } catch (const buffer::error &err) { + return -EINVAL; + } + + int r = cls_cxx_map_remove_key(hctx, mirror::peer_key(cluster_uuid)); + if (r < 0 && r != -ENOENT) { + CLS_ERR("error removing peer: %s", cpp_strerror(r).c_str()); + return r; + } + return 0; +} +/** + * Input: + * @param cluster_uuid (std::string) + * @param client_name (std::string) + * + * Output: + * @returns 0 on success, negative error code on failure + */ +int mirror_peer_set_client(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + std::string cluster_uuid; + std::string client_name; + try { + bufferlist::iterator it = in->begin(); + ::decode(cluster_uuid, it); + ::decode(client_name, it); + } catch (const buffer::error &err) { + return -EINVAL; + } + + cls::rbd::MirrorPeer peer; + int r = mirror::read_peer(hctx, cluster_uuid, &peer); + if (r < 0) { + return r; + } + + peer.client_name = client_name; + r = mirror::write_peer(hctx, cluster_uuid, peer); + if (r < 0) { + return r; + } + return 0; +} + +/** + * Input: + * @param cluster_uuid (std::string) + * @param cluster_name (std::string) + * + * Output: + * @returns 0 on success, negative error code on failure + */ +int mirror_peer_set_cluster(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + std::string cluster_uuid; + std::string cluster_name; + try { + bufferlist::iterator it = in->begin(); + ::decode(cluster_uuid, it); + ::decode(cluster_name, it); + } catch (const buffer::error &err) { + return -EINVAL; + } + + cls::rbd::MirrorPeer peer; + int r = mirror::read_peer(hctx, cluster_uuid, &peer); + if (r < 0) { + return r; + } + + peer.cluster_name = cluster_name; + r = mirror::write_peer(hctx, cluster_uuid, peer); + if (r < 0) { + return r; + } return 0; } @@ -3062,5 +3402,25 @@ void __cls_init() CLS_METHOD_RD | CLS_METHOD_WR, old_snapshot_rename, &h_old_snapshot_rename); + /* methods for the rbd_pool_settings object */ + cls_register_cxx_method(h_class, "mirror_is_enabled", CLS_METHOD_RD, + mirror_is_enabled, &h_mirror_is_enabled); + cls_register_cxx_method(h_class, "mirror_set_enabled", + CLS_METHOD_RD | CLS_METHOD_WR, + mirror_set_enabled, &h_mirror_set_enabled); + cls_register_cxx_method(h_class, "mirror_peer_list", CLS_METHOD_RD, + mirror_peer_list, &h_mirror_peer_list); + cls_register_cxx_method(h_class, "mirror_peer_add", + CLS_METHOD_RD | CLS_METHOD_WR, + mirror_peer_add, &h_mirror_peer_add); + cls_register_cxx_method(h_class, "mirror_peer_remove", + CLS_METHOD_RD | CLS_METHOD_WR, + mirror_peer_remove, &h_mirror_peer_remove); + cls_register_cxx_method(h_class, "mirror_peer_set_client", + CLS_METHOD_RD | CLS_METHOD_WR, + mirror_peer_set_client, &h_mirror_peer_set_client); + cls_register_cxx_method(h_class, "mirror_peer_set_cluster", + CLS_METHOD_RD | CLS_METHOD_WR, + mirror_peer_set_cluster, &h_mirror_peer_set_cluster); return; } diff --git a/src/cls/rbd/cls_rbd_client.cc b/src/cls/rbd/cls_rbd_client.cc index be0c256c164d..b3088d5eb004 100644 --- a/src/cls/rbd/cls_rbd_client.cc +++ b/src/cls/rbd/cls_rbd_client.cc @@ -956,5 +956,121 @@ struct C_ObjectMapLoad : public Context { return 0; } + int mirror_is_enabled(librados::IoCtx *ioctx, bool *enabled) { + bufferlist in_bl; + bufferlist out_bl; + int r = ioctx->exec(RBD_POOL_SETTINGS, "rbd", "mirror_is_enabled", in_bl, + out_bl); + if (r == -ENOENT) { + *enabled = false; + return 0; + } else if (r < 0) { + return r; + } + + try { + bufferlist::iterator bl_it = out_bl.begin(); + ::decode(*enabled, bl_it); + } catch (const buffer::error &err) { + return -EBADMSG; + } + return 0; + } + + int mirror_set_enabled(librados::IoCtx *ioctx, bool enabled) { + bufferlist in_bl; + ::encode(enabled, in_bl); + + bufferlist out_bl; + int r = ioctx->exec(RBD_POOL_SETTINGS, "rbd", "mirror_set_enabled", in_bl, + out_bl); + if (r < 0) { + return r; + } + return 0; + } + + int mirror_peer_list(librados::IoCtx *ioctx, + std::vector *peers) { + bufferlist in_bl; + bufferlist out_bl; + int r = ioctx->exec(RBD_POOL_SETTINGS, "rbd", "mirror_peer_list", in_bl, + out_bl); + if (r < 0) { + return r; + } + + peers->clear(); + try { + bufferlist::iterator bl_it = out_bl.begin(); + ::decode(*peers, bl_it); + } catch (const buffer::error &err) { + return -EBADMSG; + } + return 0; + } + + int mirror_peer_add(librados::IoCtx *ioctx, const std::string &cluster_uuid, + const std::string &cluster_name, + const std::string &client_name) { + cls::rbd::MirrorPeer peer(cluster_uuid, cluster_name, client_name); + bufferlist in_bl; + ::encode(peer, in_bl); + + bufferlist out_bl; + int r = ioctx->exec(RBD_POOL_SETTINGS, "rbd", "mirror_peer_add", in_bl, + out_bl); + if (r < 0) { + return r; + } + return 0; + } + + int mirror_peer_remove(librados::IoCtx *ioctx, + const std::string &cluster_uuid) { + bufferlist in_bl; + ::encode(cluster_uuid, in_bl); + + bufferlist out_bl; + int r = ioctx->exec(RBD_POOL_SETTINGS, "rbd", "mirror_peer_remove", in_bl, + out_bl); + if (r < 0) { + return r; + } + return 0; + } + + int mirror_peer_set_client(librados::IoCtx *ioctx, + const std::string &cluster_uuid, + const std::string &client_name) { + bufferlist in_bl; + ::encode(cluster_uuid, in_bl); + ::encode(client_name, in_bl); + + bufferlist out_bl; + int r = ioctx->exec(RBD_POOL_SETTINGS, "rbd", "mirror_peer_set_client", + in_bl, out_bl); + if (r < 0) { + return r; + } + return 0; + } + + int mirror_peer_set_cluster(librados::IoCtx *ioctx, + const std::string &cluster_uuid, + const std::string &cluster_name) { + bufferlist in_bl; + ::encode(cluster_uuid, in_bl); + ::encode(cluster_name, in_bl); + + bufferlist out_bl; + int r = ioctx->exec(RBD_POOL_SETTINGS, "rbd", "mirror_peer_set_cluster", + in_bl, out_bl); + if (r < 0) { + return r; + } + return 0; + } + } // namespace cls_client } // namespace librbd diff --git a/src/cls/rbd/cls_rbd_client.h b/src/cls/rbd/cls_rbd_client.h index 34af5427a328..a4f2dd7a186b 100644 --- a/src/cls/rbd/cls_rbd_client.h +++ b/src/cls/rbd/cls_rbd_client.h @@ -5,6 +5,7 @@ #define CEPH_LIBRBD_CLS_RBD_CLIENT_H #include "cls/lock/cls_lock_types.h" +#include "cls/rbd/cls_rbd_types.h" #include "common/bit_vector.hpp" #include "common/snap_types.h" #include "include/rados/librados.hpp" @@ -155,6 +156,24 @@ namespace librbd { std::vector *names, std::vector *sizes, ::SnapContext *snapc); + + // operations on the rbd_pool_settings object + int mirror_is_enabled(librados::IoCtx *ioctx, bool *enabled); + int mirror_set_enabled(librados::IoCtx *ioctx, bool enabled); + int mirror_peer_list(librados::IoCtx *ioctx, + std::vector *peers); + int mirror_peer_add(librados::IoCtx *ioctx, const std::string &cluster_uuid, + const std::string &cluster_name, + const std::string &client_name); + int mirror_peer_remove(librados::IoCtx *ioctx, + const std::string &cluster_uuid); + int mirror_peer_set_client(librados::IoCtx *ioctx, + const std::string &cluster_uuid, + const std::string &client_name); + int mirror_peer_set_cluster(librados::IoCtx *ioctx, + const std::string &cluster_uuid, + const std::string &cluster_name); + } // namespace cls_client } // namespace librbd #endif // CEPH_LIBRBD_CLS_RBD_CLIENT_H diff --git a/src/cls/rbd/cls_rbd_types.cc b/src/cls/rbd/cls_rbd_types.cc new file mode 100644 index 000000000000..dca84c54e658 --- /dev/null +++ b/src/cls/rbd/cls_rbd_types.cc @@ -0,0 +1,52 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "cls/rbd/cls_rbd_types.h" +#include "common/Formatter.h" + +namespace cls { +namespace rbd { + +void MirrorPeer::encode(bufferlist &bl) const { + ENCODE_START(1, 1, bl); + ::encode(cluster_uuid, bl); + ::encode(cluster_name, bl); + ::encode(client_name, bl); + ENCODE_FINISH(bl); +} + +void MirrorPeer::decode(bufferlist::iterator &it) { + DECODE_START(1, it); + ::decode(cluster_uuid, it); + ::decode(cluster_name, it); + ::decode(client_name, it); + DECODE_FINISH(it); +} + +void MirrorPeer::dump(Formatter *f) const { + f->dump_string("cluster_uuid", cluster_uuid); + f->dump_string("cluster_name", cluster_name); + f->dump_string("client_name", client_name); +} + +void MirrorPeer::generate_test_instances(std::list &o) { + o.push_back(new MirrorPeer()); + o.push_back(new MirrorPeer("uuid-123", "cluster name", "client name")); +} + +bool MirrorPeer::operator==(const MirrorPeer &rhs) const { + return (cluster_uuid == rhs.cluster_uuid && + cluster_name == rhs.cluster_name && + client_name == rhs.client_name); +} + +std::ostream& operator<<(std::ostream& os, const MirrorPeer& peer) { + os << "[" + << "cluster_uuid=" << peer.cluster_uuid << ", " + << "cluster_name=" << peer.cluster_name << ", " + << "client_name=" << peer.client_name << "]"; + return os; +} + +} // namespace rbd +} // namespace cls diff --git a/src/cls/rbd/cls_rbd_types.h b/src/cls/rbd/cls_rbd_types.h new file mode 100644 index 000000000000..a4564ce84215 --- /dev/null +++ b/src/cls/rbd/cls_rbd_types.h @@ -0,0 +1,50 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_CLS_RBD_TYPES_H +#define CEPH_CLS_RBD_TYPES_H + +#include "include/int_types.h" +#include "include/buffer.h" +#include "include/encoding.h" +#include +#include + +namespace ceph { class Formatter; } + +namespace cls { +namespace rbd { + +struct MirrorPeer { + MirrorPeer() { + } + MirrorPeer(const std::string &cluster_uuid, const std::string &cluster_name, + const std::string &client_name) + : cluster_uuid(cluster_uuid), cluster_name(cluster_name), + client_name(client_name) { + } + + std::string cluster_uuid; + std::string cluster_name; + std::string client_name; + + void encode(bufferlist &bl) const; + void decode(bufferlist::iterator &it); + void dump(Formatter *f) const; + + static void generate_test_instances(std::list &o); + + bool operator==(const MirrorPeer &rhs) const; +}; + +std::ostream& operator<<(std::ostream& os, const MirrorPeer& peer); + +WRITE_CLASS_ENCODER(MirrorPeer); + +} // namespace rbd +} // namespace cls + +using cls::rbd::encode; +using cls::rbd::decode; + +#endif // CEPH_CLS_RBD_TYPES_H diff --git a/src/include/rbd_types.h b/src/include/rbd_types.h index ad1c1b94502c..2cee7eefed04 100644 --- a/src/include/rbd_types.h +++ b/src/include/rbd_types.h @@ -58,6 +58,13 @@ #define RBD_CHILDREN "rbd_children" #define RBD_LOCK_NAME "rbd_lock" +/** + * rbd_pool_settings object in each pool contains pool-specific settings + * for configuring features such as async image mirroring to other Ceph + * clusters. + */ +#define RBD_POOL_SETTINGS "rbd_pool_settings" + #define RBD_DEFAULT_OBJ_ORDER 22 /* 4MB */ #define RBD_MAX_OBJ_NAME_SIZE 96