From: Josh Durgin Date: Fri, 12 Feb 2016 04:38:07 +0000 (-0800) Subject: cls_rbd: add methods for keeping track of mirrored images X-Git-Tag: v10.1.0~316^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=28e2d1bc55126b6c676043941a73040213a584b3;p=ceph.git cls_rbd: add methods for keeping track of mirrored images These will track whether local images should be mirrored, and map them to a unique global id. There's a state field for safely disabling mirroring while operating on multiple objects. Fixes: #14419 Signed-off-by: Josh Durgin --- diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc index 98e7be99d84..5c3ea59b4bd 100644 --- a/src/cls/rbd/cls_rbd.cc +++ b/src/cls/rbd/cls_rbd.cc @@ -118,6 +118,10 @@ cls_method_handle_t h_mirror_peer_add; cls_method_handle_t h_mirror_peer_remove; cls_method_handle_t h_mirror_peer_set_client; cls_method_handle_t h_mirror_peer_set_cluster; +cls_method_handle_t h_mirror_image_list; +cls_method_handle_t h_mirror_image_get; +cls_method_handle_t h_mirror_image_set; +cls_method_handle_t h_mirror_image_remove; #define RBD_MAX_KEYS_READ 64 #define RBD_SNAP_KEY_PREFIX "snapshot_" @@ -2927,11 +2931,16 @@ namespace mirror { static const std::string MODE("mirror_mode"); static const std::string PEER_KEY_PREFIX("mirror_peer_"); +static const std::string IMAGE_KEY_PREFIX("image_"); std::string peer_key(const std::string &uuid) { return PEER_KEY_PREFIX + uuid; } +std::string image_key(const string &image_id) { + return IMAGE_KEY_PREFIX + image_id; +} + int read_peers(cls_method_context_t hctx, std::vector *peers) { std::string last_read = PEER_KEY_PREFIX; @@ -2995,6 +3004,105 @@ int write_peer(cls_method_context_t hctx, const std::string &id, return 0; } +int image_list_ids(cls_method_context_t hctx, vector *image_ids) { + string last_read = IMAGE_KEY_PREFIX; + int max_read = RBD_MAX_KEYS_READ; + int r = max_read; + while (r == max_read) { + set keys; + r = cls_cxx_map_get_keys(hctx, last_read, max_read, &keys); + if (r < 0) { + CLS_ERR("error reading mirrored images: %s", cpp_strerror(r).c_str()); + return r; + } + + for (auto &image_key : keys) { + if (0 != image_key.compare(0, IMAGE_KEY_PREFIX.size(), IMAGE_KEY_PREFIX)) { + return 0; + } + image_ids->push_back(image_key.substr(IMAGE_KEY_PREFIX.size())); + } + } + return 0; +} + +int image_get(cls_method_context_t hctx, const string &image_id, + cls::rbd::MirrorImage *mirror_image) { + bufferlist bl; + int r = cls_cxx_map_get_val(hctx, image_key(image_id), &bl); + if (r < 0) { + if (r != -ENOENT) { + CLS_ERR("error reading mirrored image '%s': '%s'", image_id.c_str(), + cpp_strerror(r).c_str()); + } + return r; + } + + try { + bufferlist::iterator it = bl.begin(); + ::decode(*mirror_image, it); + } catch (const buffer::error &err) { + CLS_ERR("could not decode mirrored image '%s'", image_id.c_str()); + return -EIO; + } + + return 0; +} + +int image_set(cls_method_context_t hctx, const string &image_id, + const cls::rbd::MirrorImage &mirror_image) { + bufferlist bl; + ::encode(mirror_image, bl); + + // don't overwrite the key if it already exists with a different + // global_image_id + cls::rbd::MirrorImage existing_mirror_image; + int r = image_get(hctx, image_id, &existing_mirror_image); + if (r < 0 && r != -ENOENT) { + CLS_ERR("error reading mirrored image '%s': '%s'", image_id.c_str(), + cpp_strerror(r).c_str()); + return r; + } + + if (r != -ENOENT && + existing_mirror_image.global_image_id != mirror_image.global_image_id) { + return -EEXIST; + } + + r = cls_cxx_map_set_val(hctx, image_key(image_id), &bl); + if (r < 0) { + CLS_ERR("error adding mirrored image '%s': %s", image_id.c_str(), + cpp_strerror(r).c_str()); + return r; + } + return 0; +} + +int image_remove(cls_method_context_t hctx, const string &image_id) { + bufferlist bl; + cls::rbd::MirrorImage mirror_image; + int r = image_get(hctx, image_id, &mirror_image); + if (r < 0) { + if (r != -ENOENT) { + CLS_ERR("error reading mirrored image '%s': '%s'", image_id.c_str(), + cpp_strerror(r).c_str()); + } + return r; + } + + if (mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_DISABLING) { + return -EBUSY; + } + + r = cls_cxx_map_remove_key(hctx, image_key(image_id)); + if (r < 0) { + CLS_ERR("error removing mirrored image '%s': %s", image_id.c_str(), + cpp_strerror(r).c_str()); + return r; + } + return 0; +} + } // namespace mirror /** @@ -3250,6 +3358,106 @@ int mirror_peer_set_cluster(cls_method_context_t hctx, bufferlist *in, return 0; } +/** + * Input: + * none + * + * Output: + * @param std::vector: collection of image_ids + * @returns 0 on success, negative error code on failure + */ +int mirror_image_list(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + vector image_ids; + int r = mirror::image_list_ids(hctx, &image_ids); + if (r < 0) { + return r; + } + + ::encode(image_ids, *out); + return 0; +} + +/** + * Input: + * @param image_id (std::string) + * + * Output: + * @param cls::rbd::MirrorImage - metadata associated with the image_id + * @returns 0 on success, negative error code on failure + */ +int mirror_image_get(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + string image_id; + try { + bufferlist::iterator it = in->begin(); + ::decode(image_id, it); + } catch (const buffer::error &err) { + return -EINVAL; + } + + cls::rbd::MirrorImage mirror_image; + int r = mirror::image_get(hctx, image_id, &mirror_image); + if (r < 0) { + return r; + } + + ::encode(mirror_image, *out); + return 0; +} + +/** + * Input: + * @param image_id (std::string) + * @param mirror_image (cls::rbd::MirrorImage) + * + * Output: + * @returns 0 on success, negative error code on failure + * @returns -EEXIST if there's an existing image_id with a different global_image_id + */ +int mirror_image_set(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + string image_id; + cls::rbd::MirrorImage mirror_image; + try { + bufferlist::iterator it = in->begin(); + ::decode(image_id, it); + ::decode(mirror_image, it); + } catch (const buffer::error &err) { + return -EINVAL; + } + + int r = mirror::image_set(hctx, image_id, mirror_image); + if (r < 0) { + return r; + } + return 0; +} + +/** + * Input: + * @param image_id (std::string) + * + * Output: + * @returns 0 on success, negative error code on failure + */ +int mirror_image_remove(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + string image_id; + try { + bufferlist::iterator it = in->begin(); + ::decode(image_id, it); + } catch (const buffer::error &err) { + return -EINVAL; + } + + int r = mirror::image_remove(hctx, image_id); + if (r < 0) { + return r; + } + return 0; +} + void __cls_init() { CLS_LOG(20, "Loaded rbd class!"); @@ -3407,7 +3615,7 @@ void __cls_init() CLS_METHOD_RD | CLS_METHOD_WR, old_snapshot_rename, &h_old_snapshot_rename); - /* methods for the rbd_pool_settings object */ + /* methods for the rbd_mirroring object */ cls_register_cxx_method(h_class, "mirror_mode_get", CLS_METHOD_RD, mirror_mode_get, &h_mirror_mode_get); cls_register_cxx_method(h_class, "mirror_mode_set", @@ -3427,5 +3635,15 @@ void __cls_init() cls_register_cxx_method(h_class, "mirror_peer_set_cluster", CLS_METHOD_RD | CLS_METHOD_WR, mirror_peer_set_cluster, &h_mirror_peer_set_cluster); + cls_register_cxx_method(h_class, "mirror_image_list", CLS_METHOD_RD, + mirror_image_list, &h_mirror_image_list); + cls_register_cxx_method(h_class, "mirror_image_get", CLS_METHOD_RD, + mirror_image_get, &h_mirror_image_get); + cls_register_cxx_method(h_class, "mirror_image_set", + CLS_METHOD_RD | CLS_METHOD_WR, + mirror_image_set, &h_mirror_image_set); + cls_register_cxx_method(h_class, "mirror_image_remove", + CLS_METHOD_RD | CLS_METHOD_WR, + mirror_image_remove, &h_mirror_image_remove); return; } diff --git a/src/cls/rbd/cls_rbd_client.cc b/src/cls/rbd/cls_rbd_client.cc index f34c731db59..42bba3d9e01 100644 --- a/src/cls/rbd/cls_rbd_client.cc +++ b/src/cls/rbd/cls_rbd_client.cc @@ -1088,5 +1088,74 @@ namespace librbd { return 0; } + int mirror_image_list(librados::IoCtx *ioctx, + std::vector *image_ids) { + bufferlist in_bl; + bufferlist out_bl; + int r = ioctx->exec(RBD_MIRRORING, "rbd", "mirror_image_list", in_bl, + out_bl); + if (r < 0) { + return r; + } + + image_ids->clear(); + try { + bufferlist::iterator bl_it = out_bl.begin(); + ::decode(*image_ids, bl_it); + } catch (const buffer::error &err) { + return -EBADMSG; + } + return 0; + } + + int mirror_image_get(librados::IoCtx *ioctx, const std::string &image_id, + cls::rbd::MirrorImage *mirror_image) { + bufferlist in_bl; + bufferlist out_bl; + ::encode(image_id, in_bl); + + int r = ioctx->exec(RBD_MIRRORING, "rbd", "mirror_image_get", in_bl, + out_bl); + if (r < 0) { + return r; + } + + try { + bufferlist::iterator bl_it = out_bl.begin(); + ::decode(*mirror_image, bl_it); + } catch (const buffer::error &err) { + return -EBADMSG; + } + return 0; + } + + int mirror_image_set(librados::IoCtx *ioctx, const std::string &image_id, + const cls::rbd::MirrorImage &mirror_image) { + bufferlist in_bl; + ::encode(image_id, in_bl); + ::encode(mirror_image, in_bl); + + bufferlist out_bl; + int r = ioctx->exec(RBD_MIRRORING, "rbd", "mirror_image_set", in_bl, + out_bl); + if (r < 0) { + return r; + } + return 0; + } + + int mirror_image_remove(librados::IoCtx *ioctx, const std::string &image_id) { + bufferlist in_bl; + ::encode(image_id, in_bl); + + bufferlist out_bl; + int r = ioctx->exec(RBD_MIRRORING, "rbd", "mirror_image_remove", in_bl, + out_bl); + if (r < 0) { + return r; + } + return 0; + } + } // namespace cls_client } // namespace librbd diff --git a/src/cls/rbd/cls_rbd_client.h b/src/cls/rbd/cls_rbd_client.h index 2d3ac8729c1..6c7e55e4600 100644 --- a/src/cls/rbd/cls_rbd_client.h +++ b/src/cls/rbd/cls_rbd_client.h @@ -203,7 +203,7 @@ namespace librbd { std::vector *sizes, ::SnapContext *snapc); - // operations on the rbd_pool_settings object + // operations on the rbd_mirroring object int mirror_mode_get(librados::IoCtx *ioctx, cls::rbd::MirrorMode *mirror_mode); int mirror_mode_set(librados::IoCtx *ioctx, @@ -222,6 +222,14 @@ namespace librbd { int mirror_peer_set_cluster(librados::IoCtx *ioctx, const std::string &uuid, const std::string &cluster_name); + int mirror_image_list(librados::IoCtx *ioctx, + std::vector *image_ids); + int mirror_image_get(librados::IoCtx *ioctx, const std::string &image_id, + cls::rbd::MirrorImage *mirror_image); + int mirror_image_set(librados::IoCtx *ioctx, const std::string &image_id, + const cls::rbd::MirrorImage &mirror_image); + int mirror_image_remove(librados::IoCtx *ioctx, + const std::string &image_id); } // namespace cls_client } // namespace librbd diff --git a/src/cls/rbd/cls_rbd_types.cc b/src/cls/rbd/cls_rbd_types.cc index 297c2c2cb5b..3f68522468a 100644 --- a/src/cls/rbd/cls_rbd_types.cc +++ b/src/cls/rbd/cls_rbd_types.cc @@ -74,5 +74,58 @@ std::ostream& operator<<(std::ostream& os, const MirrorPeer& peer) { return os; } +void MirrorImage::encode(bufferlist &bl) const { + ENCODE_START(1, 1, bl); + ::encode(global_image_id, bl); + ::encode(static_cast(state), bl); + ENCODE_FINISH(bl); +} + +void MirrorImage::decode(bufferlist::iterator &it) { + uint8_t int_state; + DECODE_START(1, it); + ::decode(global_image_id, it); + ::decode(int_state, it); + state = static_cast(int_state); + DECODE_FINISH(it); +} + +void MirrorImage::dump(Formatter *f) const { + f->dump_string("global_image_id", global_image_id); + f->dump_int("state", state); +} + +void MirrorImage::generate_test_instances(std::list &o) { + o.push_back(new MirrorImage()); + o.push_back(new MirrorImage("uuid-123", MIRROR_IMAGE_STATE_ENABLED)); + o.push_back(new MirrorImage("uuid-abc", MIRROR_IMAGE_STATE_DISABLING)); +} + +bool MirrorImage::operator==(const MirrorImage &rhs) const { + return global_image_id == rhs.global_image_id && state == rhs.state; +} + +std::ostream& operator<<(std::ostream& os, const MirrorImageState& mirror_state) { + switch (mirror_state) { + case MIRROR_IMAGE_STATE_DISABLING: + os << "disabling"; + break; + case MIRROR_IMAGE_STATE_ENABLED: + os << "enabled"; + break; + default: + os << "unknown (" << static_cast(mirror_state) << ")"; + break; + } + return os; +} + +std::ostream& operator<<(std::ostream& os, const MirrorImage& mirror_image) { + os << "[" + << "global_image_id=" << mirror_image.global_image_id << ", " + << "state=" << mirror_image.state << "]"; + return os; +} + } // namespace rbd } // namespace cls diff --git a/src/cls/rbd/cls_rbd_types.h b/src/cls/rbd/cls_rbd_types.h index b9b1f465123..cdc81610878 100644 --- a/src/cls/rbd/cls_rbd_types.h +++ b/src/cls/rbd/cls_rbd_types.h @@ -49,6 +49,33 @@ std::ostream& operator<<(std::ostream& os, const MirrorPeer& peer); WRITE_CLASS_ENCODER(MirrorPeer); +enum MirrorImageState { + MIRROR_IMAGE_STATE_DISABLING = 0, + MIRROR_IMAGE_STATE_ENABLED = 1 +}; + +struct MirrorImage { + MirrorImage() {} + MirrorImage(const std::string &global_image_id, MirrorImageState state) + : global_image_id(global_image_id), state(state) {} + + std::string global_image_id; + MirrorImageState state; + + void encode(bufferlist &bl) const; + void decode(bufferlist::iterator &it); + void dump(Formatter *f) const; + + static void generate_test_instances(std::list &o); + + bool operator==(const MirrorImage &rhs) const; +}; + +std::ostream& operator<<(std::ostream& os, const MirrorImageState& mirror_state); +std::ostream& operator<<(std::ostream& os, const MirrorImage& mirror_image); + +WRITE_CLASS_ENCODER(MirrorImage); + } // namespace rbd } // namespace cls diff --git a/src/test/cls_rbd/test_cls_rbd.cc b/src/test/cls_rbd/test_cls_rbd.cc index f43fbdbaa3b..19d77255941 100644 --- a/src/test/cls_rbd/test_cls_rbd.cc +++ b/src/test/cls_rbd/test_cls_rbd.cc @@ -1287,6 +1287,7 @@ TEST_F(TestClsRbd, set_features) TEST_F(TestClsRbd, mirror) { librados::IoCtx ioctx; ASSERT_EQ(0, _rados.ioctx_create(_pool_name.c_str(), ioctx)); + ioctx.remove(RBD_MIRRORING); std::vector peers; ASSERT_EQ(-ENOENT, mirror_peer_list(&ioctx, &peers)); @@ -1349,3 +1350,55 @@ TEST_F(TestClsRbd, mirror) { ASSERT_EQ(0, mirror_mode_get(&ioctx, &mirror_mode)); ASSERT_EQ(cls::rbd::MIRROR_MODE_DISABLED, mirror_mode); } + +TEST_F(TestClsRbd, mirror_image) { + librados::IoCtx ioctx; + ASSERT_EQ(0, _rados.ioctx_create(_pool_name.c_str(), ioctx)); + ioctx.remove(RBD_MIRRORING); + + vector image_ids; + ASSERT_EQ(-ENOENT, mirror_image_list(&ioctx, &image_ids)); + + cls::rbd::MirrorImage image1("uuid1", cls::rbd::MIRROR_IMAGE_STATE_ENABLED); + cls::rbd::MirrorImage image2("uuid2", cls::rbd::MIRROR_IMAGE_STATE_DISABLING); + cls::rbd::MirrorImage image3("uuid3", cls::rbd::MIRROR_IMAGE_STATE_ENABLED); + + ASSERT_EQ(0, mirror_image_set(&ioctx, "image_id1", image1)); + ASSERT_EQ(0, mirror_image_set(&ioctx, "image_id2", image2)); + ASSERT_EQ(-EEXIST, mirror_image_set(&ioctx, "image_id1", image2)); + ASSERT_EQ(-EEXIST, mirror_image_set(&ioctx, "image_id2", image3)); + ASSERT_EQ(0, mirror_image_set(&ioctx, "image_id3", image3)); + + cls::rbd::MirrorImage read_image; + ASSERT_EQ(0, mirror_image_get(&ioctx, "image_id1", &read_image)); + ASSERT_EQ(read_image, image1); + ASSERT_EQ(0, mirror_image_get(&ioctx, "image_id2", &read_image)); + ASSERT_EQ(read_image, image2); + ASSERT_EQ(0, mirror_image_get(&ioctx, "image_id3", &read_image)); + ASSERT_EQ(read_image, image3); + + ASSERT_EQ(0, mirror_image_list(&ioctx, &image_ids)); + vector expected_image_ids = { + {"image_id1"}, {"image_id2"}, {"image_id3"}}; + ASSERT_EQ(expected_image_ids, image_ids); + + ASSERT_EQ(0, mirror_image_remove(&ioctx, "image_id2")); + ASSERT_EQ(-EBUSY, mirror_image_remove(&ioctx, "image_id1")); + + ASSERT_EQ(0, mirror_image_list(&ioctx, &image_ids)); + expected_image_ids = {{"image_id1"}, {"image_id3"}}; + ASSERT_EQ(expected_image_ids, image_ids); + + image1.state = cls::rbd::MIRROR_IMAGE_STATE_DISABLING; + image3.state = cls::rbd::MIRROR_IMAGE_STATE_DISABLING; + ASSERT_EQ(0, mirror_image_set(&ioctx, "image_id1", image1)); + ASSERT_EQ(0, mirror_image_get(&ioctx, "image_id1", &read_image)); + ASSERT_EQ(read_image, image1); + ASSERT_EQ(0, mirror_image_set(&ioctx, "image_id3", image3)); + ASSERT_EQ(0, mirror_image_remove(&ioctx, "image_id1")); + ASSERT_EQ(0, mirror_image_remove(&ioctx, "image_id3")); + + ASSERT_EQ(0, mirror_image_list(&ioctx, &image_ids)); + expected_image_ids = {}; + ASSERT_EQ(expected_image_ids, image_ids); +} diff --git a/src/test/encoding/types.h b/src/test/encoding/types.h index 0abef6070ee..4ffab8743cb 100644 --- a/src/test/encoding/types.h +++ b/src/test/encoding/types.h @@ -354,6 +354,7 @@ TYPE(cls_rbd_snap) #include "cls/rbd/cls_rbd_types.h" TYPE(cls::rbd::MirrorPeer) +TYPE(cls::rbd::MirrorImage) #endif #endif