From: Jason Dillaman Date: Thu, 2 Feb 2017 18:07:49 +0000 (-0500) Subject: librbd: move mirror-related API functions X-Git-Tag: v12.0.1~112^2~6 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=4aea48ec5f67938406238269339385a561c0a418;p=ceph-ci.git librbd: move mirror-related API functions Also ensure that the API helper methods can be tested via mock test cases. Signed-off-by: Jason Dillaman --- diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index ccbe4090ad2..c7b003b4667 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -22,9 +22,11 @@ set(librbd_internal_srcs ObjectMap.cc Operations.cc Utils.cc + Watcher.cc + api/Image.cc + api/Mirror.cc cache/ImageWriteback.cc cache/PassthroughImageCache.cc - Watcher.cc exclusive_lock/AutomaticPolicy.cc exclusive_lock/PreAcquireRequest.cc exclusive_lock/PostAcquireRequest.cc @@ -72,7 +74,6 @@ set(librbd_internal_srcs object_map/SnapshotRollbackRequest.cc object_map/UnlockRequest.cc object_map/UpdateRequest.cc - watcher/Notifier.cc operation/DisableFeaturesRequest.cc operation/EnableFeaturesRequest.cc operation/FlattenRequest.cc @@ -91,6 +92,7 @@ set(librbd_internal_srcs operation/SnapshotUnprotectRequest.cc operation/SnapshotLimitRequest.cc operation/TrimRequest.cc + watcher/Notifier.cc watcher/RewatchRequest.cc watcher/Types.cc ${CMAKE_SOURCE_DIR}/src/common/ContextCompletion.cc) diff --git a/src/librbd/api/Image.cc b/src/librbd/api/Image.cc new file mode 100644 index 00000000000..27049183b65 --- /dev/null +++ b/src/librbd/api/Image.cc @@ -0,0 +1,121 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/api/Image.h" +#include "include/rados/librados.hpp" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::api::Image: " << __func__ << ": " + +namespace librbd { +namespace api { + +template +int Image::list_images(librados::IoCtx& io_ctx, ImageNameToIds *images) { + CephContext *cct = (CephContext *)io_ctx.cct(); + ldout(cct, 20) << "io_ctx=" << &io_ctx << dendl; + + // new format images are accessed by class methods + int r; + int max_read = 1024; + string last_read = ""; + do { + map images_page; + r = cls_client::dir_list(&io_ctx, RBD_DIRECTORY, + last_read, max_read, &images_page); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "error listing image in directory: " + << cpp_strerror(r) << dendl; + return r; + } else if (r == -ENOENT) { + break; + } + for (map::const_iterator it = images_page.begin(); + it != images_page.end(); ++it) { + images->insert(*it); + } + if (!images_page.empty()) { + last_read = images_page.rbegin()->first; + } + r = images_page.size(); + } while (r == max_read); + + return 0; +} + +template +int Image::list_children(I *ictx, const ParentSpec &parent_spec, + PoolImageIds *pool_image_ids) +{ + CephContext *cct = ictx->cct; + int r = ictx->state->refresh_if_required(); + if (r < 0) { + return r; + } + + // no children for non-layered or old format image + if (!ictx->test_features(RBD_FEATURE_LAYERING, ictx->snap_lock)) { + return 0; + } + + pool_image_ids->clear(); + // search all pools for children depending on this snapshot + librados::Rados rados(ictx->md_ctx); + std::list > pools; + r = rados.pool_list2(pools); + if (r < 0) { + lderr(cct) << "error listing pools: " << cpp_strerror(r) << dendl; + return r; + } + + for (auto it = pools.begin(); it != pools.end(); ++it) { + int64_t base_tier; + r = rados.pool_get_base_tier(it->first, &base_tier); + if (r == -ENOENT) { + ldout(cct, 1) << "pool " << it->second << " no longer exists" << dendl; + continue; + } else if (r < 0) { + lderr(cct) << "error retrieving base tier for pool " << it->second + << dendl; + return r; + } + if (it->first != base_tier) { + // pool is a cache; skip it + continue; + } + + IoCtx ioctx; + r = rados.ioctx_create2(it->first, ioctx); + if (r == -ENOENT) { + ldout(cct, 1) << "pool " << it->second << " no longer exists" << dendl; + continue; + } else if (r < 0) { + lderr(cct) << "error accessing child image pool " << it->second + << dendl; + return r; + } + + set image_ids; + r = cls_client::get_children(&ioctx, RBD_CHILDREN, parent_spec, + image_ids); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "error reading list of children from pool " << it->second + << dendl; + return r; + } + pool_image_ids->insert({*it, image_ids}); + } + + return 0; +} + +} // namespace api +} // namespace librbd + +template class librbd::api::Image; diff --git a/src/librbd/api/Image.h b/src/librbd/api/Image.h new file mode 100644 index 00000000000..5f3dfff1443 --- /dev/null +++ b/src/librbd/api/Image.h @@ -0,0 +1,40 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef LIBRBD_API_IMAGE_H +#define LIBRBD_API_IMAGE_H + +#include "librbd/Types.h" +#include +#include +#include + +namespace librados { struct IoCtx; } + +namespace librbd { + +struct ImageCtx; + +namespace api { + +template +struct Image { + typedef std::pair PoolSpec; + typedef std::set ImageIds; + typedef std::map PoolImageIds; + typedef std::map ImageNameToIds; + + static int list_images(librados::IoCtx& io_ctx, + ImageNameToIds *images); + + static int list_children(ImageCtxT *ictx, const ParentSpec &parent_spec, + PoolImageIds *pool_image_ids); + +}; + +} // namespace api +} // namespace librbd + +extern template class librbd::api::Image; + +#endif // LIBRBD_API_IMAGE_H diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc new file mode 100644 index 00000000000..dfed5d7d38f --- /dev/null +++ b/src/librbd/api/Mirror.cc @@ -0,0 +1,895 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/api/Mirror.h" +#include "include/rados/librados.hpp" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/api/Image.h" +#include "librbd/mirror/DisableRequest.h" +#include "librbd/mirror/EnableRequest.h" +#include "librbd/MirroringWatcher.h" +#include + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::api::Mirror: " << __func__ << ": " + +namespace librbd { +namespace api { + +namespace { + +template +int validate_mirroring_enabled(I *ictx) { + CephContext *cct = ictx->cct; + cls::rbd::MirrorImage mirror_image_internal; + int r = cls_client::mirror_image_get(&ictx->md_ctx, ictx->id, + &mirror_image_internal); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + return r; + } else if (mirror_image_internal.state != + cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + lderr(cct) << "mirroring is not currently enabled" << dendl; + return -EINVAL; + } + return 0; +} + +int list_mirror_images(librados::IoCtx& io_ctx, + std::set& mirror_image_ids) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + + std::string last_read = ""; + int max_read = 1024; + int r; + do { + std::map mirror_images; + r = cls_client::mirror_image_list(&io_ctx, last_read, max_read, + &mirror_images); + if (r < 0) { + lderr(cct) << "error listing mirrored image directory: " + << cpp_strerror(r) << dendl; + return r; + } + for (auto it = mirror_images.begin(); it != mirror_images.end(); ++it) { + mirror_image_ids.insert(it->first); + } + if (!mirror_images.empty()) { + last_read = mirror_images.rbegin()->first; + } + r = mirror_images.size(); + } while (r == max_read); + + return 0; +} + +} // anonymous namespace + +template +int Mirror::image_enable(I *ictx, bool relax_same_pool_parent_check) { + CephContext *cct = ictx->cct; + ldout(cct, 20) << "ictx=" << ictx << dendl; + + int r = ictx->state->refresh_if_required(); + if (r < 0) { + return r; + } + + cls::rbd::MirrorMode mirror_mode; + r = cls_client::mirror_mode_get(&ictx->md_ctx, &mirror_mode); + if (r < 0) { + lderr(cct) << "cannot enable mirroring: failed to retrieve mirror mode: " + << cpp_strerror(r) << dendl; + return r; + } + + if (mirror_mode != cls::rbd::MIRROR_MODE_IMAGE) { + lderr(cct) << "cannot enable mirroring in the current pool mirroring mode" + << dendl; + return -EINVAL; + } + + // is mirroring not enabled for the parent? + { + RWLock::RLocker l(ictx->parent_lock); + ImageCtx *parent = ictx->parent; + if (parent) { + if (relax_same_pool_parent_check && + parent->md_ctx.get_id() == ictx->md_ctx.get_id()) { + if (!parent->test_features(RBD_FEATURE_JOURNALING)) { + lderr(cct) << "journaling is not enabled for the parent" << dendl; + return -EINVAL; + } + } else { + cls::rbd::MirrorImage mirror_image_internal; + r = cls_client::mirror_image_get(&(parent->md_ctx), parent->id, + &mirror_image_internal); + if (r == -ENOENT) { + lderr(cct) << "mirroring is not enabled for the parent" << dendl; + return -EINVAL; + } + } + } + } + + if ((ictx->features & RBD_FEATURE_JOURNALING) == 0) { + lderr(cct) << "cannot enable mirroring: journaling is not enabled" << dendl; + return -EINVAL; + } + + C_SaferCond ctx; + auto req = mirror::EnableRequest::create(ictx, &ctx); + req->send(); + + r = ctx.wait(); + if (r < 0) { + lderr(cct) << "cannot enable mirroring: " << cpp_strerror(r) << dendl; + return r; + } + + return 0; +} + +template +int Mirror::image_disable(I *ictx, bool force) { + CephContext *cct = ictx->cct; + ldout(cct, 20) << "ictx=" << ictx << dendl; + + int r = ictx->state->refresh_if_required(); + if (r < 0) { + return r; + } + + cls::rbd::MirrorMode mirror_mode; + r = cls_client::mirror_mode_get(&ictx->md_ctx, &mirror_mode); + if (r < 0) { + lderr(cct) << "cannot disable mirroring: failed to retrieve pool " + "mirroring mode: " << cpp_strerror(r) << dendl; + return r; + } + + if (mirror_mode != cls::rbd::MIRROR_MODE_IMAGE) { + lderr(cct) << "cannot disable mirroring in the current pool mirroring " + "mode" << dendl; + return -EINVAL; + } + + // is mirroring enabled for the child? + cls::rbd::MirrorImage mirror_image_internal; + r = cls_client::mirror_image_get(&ictx->md_ctx, ictx->id, + &mirror_image_internal); + if (r == -ENOENT) { + // mirroring is not enabled for this image + ldout(cct, 20) << "ignoring disable command: mirroring is not enabled for " + << "this image" << dendl; + return 0; + } else if (r == -EOPNOTSUPP) { + ldout(cct, 5) << "mirroring not supported by OSD" << dendl; + return r; + } else if (r < 0) { + lderr(cct) << "failed to retrieve mirror image metadata: " + << cpp_strerror(r) << dendl; + return r; + } + + mirror_image_internal.state = cls::rbd::MIRROR_IMAGE_STATE_DISABLING; + r = cls_client::mirror_image_set(&ictx->md_ctx, ictx->id, + mirror_image_internal); + if (r < 0) { + lderr(cct) << "cannot disable mirroring: " << cpp_strerror(r) << dendl; + return r; + } else { + bool rollback = false; + BOOST_SCOPE_EXIT_ALL(ictx, &mirror_image_internal, &rollback) { + if (rollback) { + CephContext *cct = ictx->cct; + mirror_image_internal.state = cls::rbd::MIRROR_IMAGE_STATE_ENABLED; + int r = cls_client::mirror_image_set(&ictx->md_ctx, ictx->id, + mirror_image_internal); + if (r < 0) { + lderr(cct) << "failed to re-enable image mirroring: " + << cpp_strerror(r) << dendl; + } + } + }; + + { + RWLock::RLocker l(ictx->snap_lock); + map snap_info = ictx->snap_info; + for (auto &info : snap_info) { + ParentSpec parent_spec(ictx->md_ctx.get_id(), ictx->id, info.first); + map< pair, set > image_info; + + r = Image::list_children(ictx, parent_spec, &image_info); + if (r < 0) { + rollback = true; + return r; + } + if (image_info.empty()) + continue; + + librados::Rados rados(ictx->md_ctx); + for (auto &info: image_info) { + librados::IoCtx ioctx; + r = rados.ioctx_create2(info.first.first, ioctx); + if (r < 0) { + rollback = true; + lderr(cct) << "error accessing child image pool " + << info.first.second << dendl; + return r; + } + for (auto &id_it : info.second) { + cls::rbd::MirrorImage mirror_image_internal; + r = cls_client::mirror_image_get(&ioctx, id_it, + &mirror_image_internal); + if (r != -ENOENT) { + rollback = true; + lderr(cct) << "mirroring is enabled on one or more children " + << dendl; + return -EBUSY; + } + } + } + } + } + + C_SaferCond ctx; + auto req = mirror::DisableRequest::create(ictx, force, true, + &ctx); + req->send(); + + r = ctx.wait(); + if (r < 0) { + lderr(cct) << "cannot disable mirroring: " << cpp_strerror(r) << dendl; + rollback = true; + return r; + } + } + + return 0; +} + +template +int Mirror::image_promote(I *ictx, bool force) { + CephContext *cct = ictx->cct; + ldout(cct, 20) << "ictx=" << ictx << ", " + << "force=" << force << dendl; + + int r = ictx->state->refresh_if_required(); + if (r < 0) { + return r; + } + + r = validate_mirroring_enabled(ictx); + if (r < 0) { + return r; + } + + std::string mirror_uuid; + r = Journal::get_tag_owner(ictx, &mirror_uuid); + if (r < 0) { + lderr(cct) << "failed to determine tag ownership: " << cpp_strerror(r) + << dendl; + return r; + } else if (mirror_uuid == Journal<>::LOCAL_MIRROR_UUID) { + lderr(cct) << "image is already primary" << dendl; + return -EINVAL; + } else if (mirror_uuid != Journal<>::ORPHAN_MIRROR_UUID && !force) { + lderr(cct) << "image is still primary within a remote cluster" << dendl; + return -EBUSY; + } + + // TODO: need interlock with local rbd-mirror daemon to ensure it has stopped + // replay + + r = Journal::promote(ictx); + if (r < 0) { + lderr(cct) << "failed to promote image: " << cpp_strerror(r) + << dendl; + return r; + } + return 0; +} + +template +int Mirror::image_demote(I *ictx) { + CephContext *cct = ictx->cct; + ldout(cct, 20) << "ictx=" << ictx << dendl; + + int r = ictx->state->refresh_if_required(); + if (r < 0) { + return r; + } + + r = validate_mirroring_enabled(ictx); + if (r < 0) { + return r; + } + + bool is_primary; + r = Journal::is_tag_owner(ictx, &is_primary); + if (r < 0) { + lderr(cct) << "failed to determine tag ownership: " << cpp_strerror(r) + << dendl; + return r; + } + + if (!is_primary) { + lderr(cct) << "image is not currently the primary" << dendl; + return -EINVAL; + } + + RWLock::RLocker owner_lock(ictx->owner_lock); + if (ictx->exclusive_lock == nullptr) { + lderr(cct) << "exclusive lock is not active" << dendl; + return -EINVAL; + } + + // avoid accepting new requests from peers while we demote + // the image + ictx->exclusive_lock->block_requests(0); + BOOST_SCOPE_EXIT_ALL( (ictx) ) { + if (ictx->exclusive_lock != nullptr) { + ictx->exclusive_lock->unblock_requests(); + } + }; + + C_SaferCond lock_ctx; + ictx->exclusive_lock->acquire_lock(&lock_ctx); + + // don't block holding lock since refresh might be required + ictx->owner_lock.put_read(); + r = lock_ctx.wait(); + ictx->owner_lock.get_read(); + + if (r < 0) { + lderr(cct) << "failed to lock image: " << cpp_strerror(r) << dendl; + return r; + } else if (ictx->exclusive_lock == nullptr || + !ictx->exclusive_lock->is_lock_owner()) { + lderr(cct) << "failed to acquire exclusive lock" << dendl; + return -EROFS; + } + + BOOST_SCOPE_EXIT_ALL( (ictx) ) { + C_SaferCond lock_ctx; + ictx->exclusive_lock->release_lock(&lock_ctx); + lock_ctx.wait(); + }; + + RWLock::RLocker snap_locker(ictx->snap_lock); + if (ictx->journal == nullptr) { + lderr(cct) << "journal is not active" << dendl; + return -EINVAL; + } else if (!ictx->journal->is_tag_owner()) { + lderr(cct) << "image is not currently the primary" << dendl; + return -EINVAL; + } + + r = ictx->journal->demote(); + if (r < 0) { + lderr(cct) << "failed to demote image: " << cpp_strerror(r) + << dendl; + return r; + } + return 0; +} + +template +int Mirror::image_resync(I *ictx) { + CephContext *cct = ictx->cct; + ldout(cct, 20) << "ictx=" << ictx << dendl; + + int r = ictx->state->refresh_if_required(); + if (r < 0) { + return r; + } + + r = validate_mirroring_enabled(ictx); + if (r < 0) { + return r; + } + + std::string mirror_uuid; + r = Journal::get_tag_owner(ictx, &mirror_uuid); + if (r < 0) { + lderr(cct) << "failed to determine tag ownership: " << cpp_strerror(r) + << dendl; + return r; + } else if (mirror_uuid == Journal<>::LOCAL_MIRROR_UUID) { + lderr(cct) << "image is primary, cannot resync to itself" << dendl; + return -EINVAL; + } + + // flag the journal indicating that we want to rebuild the local image + r = Journal::request_resync(ictx); + if (r < 0) { + lderr(cct) << "failed to request resync: " << cpp_strerror(r) << dendl; + return r; + } + + return 0; +} + +template +int Mirror::image_get_info(I *ictx, mirror_image_info_t *mirror_image_info, + size_t info_size) { + CephContext *cct = ictx->cct; + ldout(cct, 20) << "ictx=" << ictx << dendl; + if (info_size < sizeof(mirror_image_info_t)) { + return -ERANGE; + } + + int r = ictx->state->refresh_if_required(); + if (r < 0) { + return r; + } + + cls::rbd::MirrorImage mirror_image_internal; + r = cls_client::mirror_image_get(&ictx->md_ctx, ictx->id, + &mirror_image_internal); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + return r; + } + + mirror_image_info->global_id = mirror_image_internal.global_image_id; + if (r == -ENOENT) { + mirror_image_info->state = RBD_MIRROR_IMAGE_DISABLED; + } else { + mirror_image_info->state = + static_cast(mirror_image_internal.state); + } + + if (mirror_image_info->state == RBD_MIRROR_IMAGE_ENABLED) { + r = Journal::is_tag_owner(ictx, &mirror_image_info->primary); + if (r < 0) { + lderr(cct) << "failed to check tag ownership: " + << cpp_strerror(r) << dendl; + return r; + } + } else { + mirror_image_info->primary = false; + } + + return 0; +} + +template +int Mirror::image_get_status(I *ictx, mirror_image_status_t *status, + size_t status_size) { + CephContext *cct = ictx->cct; + ldout(cct, 20) << "ictx=" << ictx << dendl; + if (status_size < sizeof(mirror_image_status_t)) { + return -ERANGE; + } + + int r = ictx->state->refresh_if_required(); + if (r < 0) { + return r; + } + + mirror_image_info_t info; + r = image_get_info(ictx, &info, sizeof(info)); + if (r < 0) { + return r; + } + + cls::rbd::MirrorImageStatus + s(cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN, "status not found"); + + r = cls_client::mirror_image_status_get(&ictx->md_ctx, info.global_id, &s); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve image mirror status: " + << cpp_strerror(r) << dendl; + return r; + } + + *status = mirror_image_status_t{ + ictx->name, + info, + static_cast(s.state), + s.description, + s.last_update.sec(), + s.up}; + return 0; +} + +template +int Mirror::mode_get(librados::IoCtx& io_ctx, + rbd_mirror_mode_t *mirror_mode) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << dendl; + + cls::rbd::MirrorMode mirror_mode_internal; + int r = cls_client::mirror_mode_get(&io_ctx, &mirror_mode_internal); + if (r < 0) { + lderr(cct) << "failed to retrieve mirror mode: " << cpp_strerror(r) + << dendl; + return r; + } + + switch (mirror_mode_internal) { + case cls::rbd::MIRROR_MODE_DISABLED: + case cls::rbd::MIRROR_MODE_IMAGE: + case cls::rbd::MIRROR_MODE_POOL: + *mirror_mode = static_cast(mirror_mode_internal); + break; + default: + lderr(cct) << "unknown mirror mode (" + << static_cast(mirror_mode_internal) << ")" + << dendl; + return -EINVAL; + } + return 0; +} + +template +int Mirror::mode_set(librados::IoCtx& io_ctx, + rbd_mirror_mode_t mirror_mode) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << dendl; + + cls::rbd::MirrorMode next_mirror_mode; + switch (mirror_mode) { + case RBD_MIRROR_MODE_DISABLED: + case RBD_MIRROR_MODE_IMAGE: + case RBD_MIRROR_MODE_POOL: + next_mirror_mode = static_cast(mirror_mode); + break; + default: + lderr(cct) << "unknown mirror mode (" + << static_cast(mirror_mode) << ")" << dendl; + return -EINVAL; + } + + int r; + if (next_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) { + // fail early if pool still has peers registered and attempting to disable + std::vector mirror_peers; + r = cls_client::mirror_peer_list(&io_ctx, &mirror_peers); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to list peers: " << cpp_strerror(r) << dendl; + return r; + } else if (!mirror_peers.empty()) { + lderr(cct) << "mirror peers still registered" << dendl; + return -EBUSY; + } + } + + cls::rbd::MirrorMode current_mirror_mode; + r = cls_client::mirror_mode_get(&io_ctx, ¤t_mirror_mode); + if (r < 0) { + lderr(cct) << "failed to retrieve mirror mode: " << cpp_strerror(r) + << dendl; + return r; + } + + if (current_mirror_mode == next_mirror_mode) { + return 0; + } else if (current_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) { + uuid_d uuid_gen; + uuid_gen.generate_random(); + r = cls_client::mirror_uuid_set(&io_ctx, uuid_gen.to_string()); + if (r < 0) { + lderr(cct) << "failed to allocate mirroring uuid: " << cpp_strerror(r) + << dendl; + return r; + } + } + + if (current_mirror_mode != cls::rbd::MIRROR_MODE_IMAGE) { + r = cls_client::mirror_mode_set(&io_ctx, cls::rbd::MIRROR_MODE_IMAGE); + if (r < 0) { + lderr(cct) << "failed to set mirror mode to image: " + << cpp_strerror(r) << dendl; + return r; + } + + r = MirroringWatcher<>::notify_mode_updated(io_ctx, + cls::rbd::MIRROR_MODE_IMAGE); + if (r < 0) { + lderr(cct) << "failed to send update notification: " << cpp_strerror(r) + << dendl; + } + } + + if (next_mirror_mode == cls::rbd::MIRROR_MODE_IMAGE) { + return 0; + } + + if (next_mirror_mode == cls::rbd::MIRROR_MODE_POOL) { + map images; + r = Image::list_images(io_ctx, &images); + if (r < 0) { + lderr(cct) << "failed listing images: " << cpp_strerror(r) << dendl; + return r; + } + + for (const auto& img_pair : images) { + uint64_t features; + r = cls_client::get_features(&io_ctx, + util::header_name(img_pair.second), + CEPH_NOSNAP, &features); + if (r < 0) { + lderr(cct) << "error getting features for image " << img_pair.first + << ": " << cpp_strerror(r) << dendl; + return r; + } + + if ((features & RBD_FEATURE_JOURNALING) != 0) { + I *img_ctx = I::create("", img_pair.second, nullptr, io_ctx, false); + r = img_ctx->state->open(false); + if (r < 0) { + lderr(cct) << "error opening image "<< img_pair.first << ": " + << cpp_strerror(r) << dendl; + return r; + } + + r = image_enable(img_ctx, true); + int close_r = img_ctx->state->close(); + if (r < 0) { + lderr(cct) << "error enabling mirroring for image " + << img_pair.first << ": " << cpp_strerror(r) << dendl; + return r; + } else if (close_r < 0) { + lderr(cct) << "failed to close image " << img_pair.first << ": " + << cpp_strerror(close_r) << dendl; + return close_r; + } + } + } + } else if (next_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) { + std::set image_ids; + r = list_mirror_images(io_ctx, image_ids); + if (r < 0) { + lderr(cct) << "failed listing images: " << cpp_strerror(r) << dendl; + return r; + } + + for (const auto& img_id : image_ids) { + if (current_mirror_mode == cls::rbd::MIRROR_MODE_IMAGE) { + cls::rbd::MirrorImage mirror_image; + r = cls_client::mirror_image_get(&io_ctx, img_id, &mirror_image); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve mirroring state for image id " + << img_id << ": " << cpp_strerror(r) << dendl; + return r; + } + if (mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + lderr(cct) << "failed to disable mirror mode: there are still " + << "images with mirroring enabled" << dendl; + return -EINVAL; + } + } else { + I *img_ctx = I::create("", img_id, nullptr, io_ctx, false); + r = img_ctx->state->open(false); + if (r < 0) { + lderr(cct) << "error opening image id "<< img_id << ": " + << cpp_strerror(r) << dendl; + return r; + } + + r = image_disable(img_ctx, false); + int close_r = img_ctx->state->close(); + if (r < 0) { + lderr(cct) << "error disabling mirroring for image id " << img_id + << cpp_strerror(r) << dendl; + return r; + } else if (close_r < 0) { + lderr(cct) << "failed to close image id " << img_id << ": " + << cpp_strerror(close_r) << dendl; + return close_r; + } + } + } + } + + r = cls_client::mirror_mode_set(&io_ctx, next_mirror_mode); + if (r < 0) { + lderr(cct) << "failed to set mirror mode: " << cpp_strerror(r) << dendl; + return r; + } + + r = MirroringWatcher<>::notify_mode_updated(io_ctx, next_mirror_mode); + if (r < 0) { + lderr(cct) << "failed to send update notification: " << cpp_strerror(r) + << dendl; + } + return 0; +} + +template +int Mirror::peer_add(librados::IoCtx& io_ctx, std::string *uuid, + const std::string &cluster_name, + const std::string &client_name) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << "name=" << cluster_name << ", " + << "client=" << client_name << dendl; + + if (cct->_conf->cluster == cluster_name) { + lderr(cct) << "cannot add self as remote peer" << dendl; + return -EINVAL; + } + + int r; + do { + uuid_d uuid_gen; + uuid_gen.generate_random(); + + *uuid = uuid_gen.to_string(); + r = cls_client::mirror_peer_add(&io_ctx, *uuid, cluster_name, + client_name); + if (r == -ESTALE) { + ldout(cct, 5) << "duplicate UUID detected, retrying" << dendl; + } else if (r < 0) { + lderr(cct) << "failed to add mirror peer '" << uuid << "': " + << cpp_strerror(r) << dendl; + return r; + } + } while (r == -ESTALE); + return 0; +} + +template +int Mirror::peer_remove(librados::IoCtx& io_ctx, const std::string &uuid) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << "uuid=" << uuid << dendl; + + int r = cls_client::mirror_peer_remove(&io_ctx, uuid); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to remove peer '" << uuid << "': " + << cpp_strerror(r) << dendl; + return r; + } + return 0; +} + +template +int Mirror::peer_list(librados::IoCtx& io_ctx, + std::vector *peers) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << dendl; + + std::vector mirror_peers; + int r = cls_client::mirror_peer_list(&io_ctx, &mirror_peers); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to list peers: " << cpp_strerror(r) << dendl; + return r; + } + + peers->clear(); + peers->reserve(mirror_peers.size()); + for (auto &mirror_peer : mirror_peers) { + mirror_peer_t peer; + peer.uuid = mirror_peer.uuid; + peer.cluster_name = mirror_peer.cluster_name; + peer.client_name = mirror_peer.client_name; + peers->push_back(peer); + } + return 0; +} + +template +int Mirror::peer_set_client(librados::IoCtx& io_ctx, const std::string &uuid, + const std::string &client_name) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << "uuid=" << uuid << ", " + << "client=" << client_name << dendl; + + int r = cls_client::mirror_peer_set_client(&io_ctx, uuid, client_name); + if (r < 0) { + lderr(cct) << "failed to update client '" << uuid << "': " + << cpp_strerror(r) << dendl; + return r; + } + return 0; +} + +template +int Mirror::peer_set_cluster(librados::IoCtx& io_ctx, + const std::string &uuid, + const std::string &cluster_name) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << "uuid=" << uuid << ", " + << "cluster=" << cluster_name << dendl; + + int r = cls_client::mirror_peer_set_cluster(&io_ctx, uuid, cluster_name); + if (r < 0) { + lderr(cct) << "failed to update cluster '" << uuid << "': " + << cpp_strerror(r) << dendl; + return r; + } + return 0; +} + +template +int Mirror::image_status_list(librados::IoCtx& io_ctx, + const std::string &start_id, size_t max, + IdToMirrorImageStatus *images) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + int r; + + map id_to_name; + { + map name_to_id; + r = Image::list_images(io_ctx, &name_to_id); + if (r < 0) { + return r; + } + for (auto it : name_to_id) { + id_to_name[it.second] = it.first; + } + } + + map images_; + map statuses_; + + r = librbd::cls_client::mirror_image_status_list(&io_ctx, start_id, max, + &images_, &statuses_); + if (r < 0) { + lderr(cct) << "failed to list mirror image statuses: " + << cpp_strerror(r) << dendl; + return r; + } + + cls::rbd::MirrorImageStatus unknown_status( + cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN, "status not found"); + + for (auto it = images_.begin(); it != images_.end(); ++it) { + auto &image_id = it->first; + auto &info = it->second; + auto &image_name = id_to_name[image_id]; + if (image_name.empty()) { + lderr(cct) << "failed to find image name for image " << image_id << ", " + << "using image id as name" << dendl; + image_name = image_id; + } + auto s_it = statuses_.find(image_id); + auto &s = s_it != statuses_.end() ? s_it->second : unknown_status; + (*images)[image_id] = mirror_image_status_t{ + image_name, + mirror_image_info_t{ + info.global_image_id, + static_cast(info.state), + false}, // XXX: To set "primary" right would require an additional call. + static_cast(s.state), + s.description, + s.last_update.sec(), + s.up}; + } + + return 0; +} + +template +int Mirror::image_status_summary(librados::IoCtx& io_ctx, + MirrorImageStatusStates *states) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + + std::map states_; + int r = cls_client::mirror_image_status_get_summary(&io_ctx, &states_); + if (r < 0) { + lderr(cct) << "failed to get mirror status summary: " + << cpp_strerror(r) << dendl; + return r; + } + for (auto &s : states_) { + (*states)[static_cast(s.first)] = s.second; + } + return 0; +} + +} // namespace api +} // namespace librbd + +template class librbd::api::Mirror; diff --git a/src/librbd/api/Mirror.h b/src/librbd/api/Mirror.h new file mode 100644 index 00000000000..7a8b9f3a287 --- /dev/null +++ b/src/librbd/api/Mirror.h @@ -0,0 +1,61 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef LIBRBD_API_MIRROR_H +#define LIBRBD_API_MIRROR_H + +#include "include/rbd/librbd.hpp" +#include +#include +#include + +namespace librbd { + +struct ImageCtx; + +namespace api { + +template +struct Mirror { + typedef std::map IdToMirrorImageStatus; + typedef std::map MirrorImageStatusStates; + + static int mode_get(librados::IoCtx& io_ctx, rbd_mirror_mode_t *mirror_mode); + static int mode_set(librados::IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode); + + static int peer_add(librados::IoCtx& io_ctx, std::string *uuid, + const std::string &cluster_name, + const std::string &client_name); + static int peer_remove(librados::IoCtx& io_ctx, const std::string &uuid); + static int peer_list(librados::IoCtx& io_ctx, + std::vector *peers); + static int peer_set_client(librados::IoCtx& io_ctx, const std::string &uuid, + const std::string &client_name); + static int peer_set_cluster(librados::IoCtx& io_ctx, const std::string &uuid, + const std::string &cluster_name); + + static int image_status_list(librados::IoCtx& io_ctx, + const std::string &start_id, size_t max, + IdToMirrorImageStatus *images); + static int image_status_summary(librados::IoCtx& io_ctx, + MirrorImageStatusStates *states); + + static int image_enable(ImageCtxT *ictx, bool relax_same_pool_parent_check); + static int image_disable(ImageCtxT *ictx, bool force); + static int image_promote(ImageCtxT *ictx, bool force); + static int image_demote(ImageCtxT *ictx); + static int image_resync(ImageCtxT *ictx); + static int image_get_info(ImageCtxT *ictx, + mirror_image_info_t *mirror_image_info, + size_t info_size); + static int image_get_status(ImageCtxT *ictx, mirror_image_status_t *status, + size_t status_size); + +}; + +} // namespace api +} // namespace librbd + +extern template class librbd::api::Mirror; + +#endif // LIBRBD_API_MIRROR_H diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index f32d0935d05..c7b6611aba1 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -27,24 +27,23 @@ #include "librbd/ImageState.h" #include "librbd/internal.h" #include "librbd/Journal.h" -#include "librbd/MirroringWatcher.h" #include "librbd/ObjectMap.h" #include "librbd/Operations.h" #include "librbd/Types.h" #include "librbd/Utils.h" +#include "librbd/api/Image.h" #include "librbd/exclusive_lock/AutomaticPolicy.h" #include "librbd/exclusive_lock/StandardPolicy.h" #include "librbd/image/CreateRequest.h" #include "librbd/image/RemoveRequest.h" -#include "librbd/managed_lock/Types.h" -#include "librbd/mirror/DisableRequest.h" -#include "librbd/mirror/EnableRequest.h" #include "librbd/io/AioCompletion.h" #include "librbd/io/ImageRequest.h" #include "librbd/io/ImageRequestWQ.h" #include "librbd/io/ObjectRequest.h" #include "librbd/io/ReadResult.h" #include "librbd/journal/Types.h" +#include "librbd/managed_lock/Types.h" +#include "librbd/mirror/EnableRequest.h" #include "librbd/operation/TrimRequest.h" #include "journal/Journaler.h" @@ -110,61 +109,6 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) { return 0; } -int validate_mirroring_enabled(ImageCtx *ictx) { - CephContext *cct = ictx->cct; - cls::rbd::MirrorImage mirror_image_internal; - int r = cls_client::mirror_image_get(&ictx->md_ctx, ictx->id, - &mirror_image_internal); - if (r < 0 && r != -ENOENT) { - lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) - << dendl; - return r; - } else if (mirror_image_internal.state != - cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { - lderr(cct) << "mirroring is not currently enabled" << dendl; - return -EINVAL; - } - return 0; -} - -int mirror_image_enable_internal(ImageCtx *ictx) { - CephContext *cct = ictx->cct; - C_SaferCond cond; - - if ((ictx->features & RBD_FEATURE_JOURNALING) == 0) { - lderr(cct) << "cannot enable mirroring: journaling is not enabled" << dendl; - return -EINVAL; - } - - mirror::EnableRequest *req = - mirror::EnableRequest::create(ictx, &cond); - req->send(); - - int r = cond.wait(); - if (r < 0) { - lderr(cct) << "cannot enable mirroring: " << cpp_strerror(r) << dendl; - return r; - } - - return 0; -} - -int mirror_image_disable_internal(ImageCtx *ictx, bool force, - bool remove=true) { - CephContext *cct = ictx->cct; - C_SaferCond cond; - - mirror::DisableRequest *req = - mirror::DisableRequest::create(ictx, force, remove, &cond); - req->send(); - - int r = cond.wait(); - if (r < 0) { - lderr(cct) << "cannot disable mirroring: " << cpp_strerror(r) << dendl; - return r; - } - return 0; -} } // anonymous namespace @@ -570,38 +514,6 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force, return (*opts_)->empty(); } - int list_images_v2(IoCtx& io_ctx, map &images) { - CephContext *cct = (CephContext *)io_ctx.cct(); - ldout(cct, 20) << "list_images_v2 " << &io_ctx << dendl; - - // new format images are accessed by class methods - int r; - int max_read = 1024; - string last_read = ""; - do { - map images_page; - r = cls_client::dir_list(&io_ctx, RBD_DIRECTORY, - last_read, max_read, &images_page); - if (r < 0 && r != -ENOENT) { - lderr(cct) << "error listing image in directory: " - << cpp_strerror(r) << dendl; - return r; - } else if (r == -ENOENT) { - break; - } - for (map::const_iterator it = images_page.begin(); - it != images_page.end(); ++it) { - images.insert(*it); - } - if (!images_page.empty()) { - last_read = images_page.rbegin()->first; - } - r = images_page.size(); - } while (r == max_read); - - return 0; - } - int list(IoCtx& io_ctx, vector& names) { CephContext *cct = (CephContext *)io_ctx.cct(); @@ -625,7 +537,7 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force, } map images; - r = list_images_v2(io_ctx, images); + r = api::Image<>::list_images(io_ctx, &images); if (r < 0) { lderr(cct) << "error listing v2 images: " << cpp_strerror(r) << dendl; return r; @@ -637,7 +549,8 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force, return 0; } - int flatten_children(ImageCtx *ictx, const char* snap_name, ProgressContext& pctx) + int flatten_children(ImageCtx *ictx, const char* snap_name, + ProgressContext& pctx) { CephContext *cct = ictx->cct; ldout(cct, 20) << "children flatten " << ictx->name << dendl; @@ -647,7 +560,7 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force, ParentSpec parent_spec(ictx->md_ctx.get_id(), ictx->id, snap_id); map< pair, set > image_info; - int r = list_children_info(ictx, parent_spec, image_info); + int r = api::Image<>::list_children(ictx, parent_spec, &image_info); if (r < 0) { return r; } @@ -723,7 +636,7 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force, ParentSpec parent_spec(ictx->md_ctx.get_id(), ictx->id, ictx->snap_id); map< pair, set > image_info; - int r = list_children_info(ictx, parent_spec, image_info); + int r = api::Image<>::list_children(ictx, parent_spec, &image_info); if (r < 0) { return r; } @@ -753,70 +666,6 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force, return 0; } - int list_children_info(ImageCtx *ictx, const ParentSpec &parent_spec, - map< pair, set >& image_info) - { - CephContext *cct = ictx->cct; - int r = ictx->state->refresh_if_required(); - if (r < 0) - return r; - - // no children for non-layered or old format image - if (!ictx->test_features(RBD_FEATURE_LAYERING, ictx->snap_lock)) - return 0; - - image_info.clear(); - // search all pools for children depending on this snapshot - Rados rados(ictx->md_ctx); - std::list > pools; - r = rados.pool_list2(pools); - if (r < 0) { - lderr(cct) << "error listing pools: " << cpp_strerror(r) << dendl; - return r; - } - - for (std::list >::const_iterator it = - pools.begin(); it != pools.end(); ++it) { - int64_t base_tier; - r = rados.pool_get_base_tier(it->first, &base_tier); - if (r == -ENOENT) { - ldout(cct, 1) << "pool " << it->second << " no longer exists" << dendl; - continue; - } else if (r < 0) { - lderr(cct) << "Error retrieving base tier for pool " << it->second - << dendl; - return r; - } - if (it->first != base_tier) { - // pool is a cache; skip it - continue; - } - - IoCtx ioctx; - r = rados.ioctx_create2(it->first, ioctx); - if (r == -ENOENT) { - ldout(cct, 1) << "pool " << it->second << " no longer exists" << dendl; - continue; - } else if (r < 0) { - lderr(cct) << "Error accessing child image pool " << it->second - << dendl; - return r; - } - - set image_ids; - r = cls_client::get_children(&ioctx, RBD_CHILDREN, parent_spec, - image_ids); - if (r < 0 && r != -ENOENT) { - lderr(cct) << "Error reading list of children from pool " << it->second - << dendl; - return r; - } - image_info.insert(make_pair(make_pair(it->first, it->second), image_ids)); - } - - return 0; - } - int get_snap_namespace(ImageCtx *ictx, const char *snap_name, cls::rbd::SnapshotNamespace *snap_namespace) { @@ -2305,809 +2154,6 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force, return cls_client::metadata_list(&ictx->md_ctx, ictx->header_oid, start, max, pairs); } - int mirror_image_enable(ImageCtx *ictx, bool relax_same_pool_parent_check) { - CephContext *cct = ictx->cct; - ldout(cct, 20) << "mirror_image_enable " << ictx << dendl; - - int r = ictx->state->refresh_if_required(); - if (r < 0) { - return r; - } - - cls::rbd::MirrorMode mirror_mode; - r = cls_client::mirror_mode_get(&ictx->md_ctx, &mirror_mode); - if (r < 0) { - lderr(cct) << "cannot enable mirroring: failed to retrieve mirror mode: " - << cpp_strerror(r) << dendl; - return r; - } - - if (mirror_mode != cls::rbd::MIRROR_MODE_IMAGE) { - lderr(cct) << "cannot enable mirroring in the current pool mirroring " - "mode" << dendl; - return -EINVAL; - } - - // is mirroring not enabled for the parent? - { - RWLock::RLocker l(ictx->parent_lock); - ImageCtx *parent = ictx->parent; - if (parent) { - if (relax_same_pool_parent_check && - parent->md_ctx.get_id() == ictx->md_ctx.get_id()) { - if (!parent->test_features(RBD_FEATURE_JOURNALING)) { - lderr(cct) << "journaling is not enabled for the parent" << dendl; - return -EINVAL; - } - } else { - cls::rbd::MirrorImage mirror_image_internal; - r = cls_client::mirror_image_get(&(parent->md_ctx), parent->id, - &mirror_image_internal); - if (r == -ENOENT) { - lderr(cct) << "mirroring is not enabled for the parent" << dendl; - return -EINVAL; - } - } - } - } - - r = mirror_image_enable_internal(ictx); - if (r < 0) { - return r; - } - return 0; - } - - int mirror_image_disable(ImageCtx *ictx, bool force) { - CephContext *cct = ictx->cct; - ldout(cct, 20) << "mirror_image_disable " << ictx << dendl; - - int r = ictx->state->refresh_if_required(); - if (r < 0) { - return r; - } - - cls::rbd::MirrorMode mirror_mode; - r = cls_client::mirror_mode_get(&ictx->md_ctx, &mirror_mode); - if (r < 0) { - lderr(cct) << "cannot disable mirroring: failed to retrieve pool " - "mirroring mode: " << cpp_strerror(r) << dendl; - return r; - } - - if (mirror_mode != cls::rbd::MIRROR_MODE_IMAGE) { - lderr(cct) << "cannot disable mirroring in the current pool mirroring " - "mode" << dendl; - return -EINVAL; - } - - // is mirroring enabled for the child? - cls::rbd::MirrorImage mirror_image_internal; - r = cls_client::mirror_image_get(&ictx->md_ctx, ictx->id, &mirror_image_internal); - if (r == -ENOENT) { - // mirroring is not enabled for this image - ldout(cct, 20) << "ignoring disable command: mirroring is not enabled for this image" - << dendl; - return 0; - } else if (r == -EOPNOTSUPP) { - ldout(cct, 5) << "mirroring not supported by OSD" << dendl; - return r; - } else if (r < 0) { - lderr(cct) << "failed to retrieve mirror image metadata: " << cpp_strerror(r) << dendl; - return r; - } - mirror_image_internal.state = cls::rbd::MIRROR_IMAGE_STATE_DISABLING; - r = cls_client::mirror_image_set(&ictx->md_ctx, ictx->id, mirror_image_internal); - if (r < 0) { - lderr(cct) << "cannot disable mirroring: " << cpp_strerror(r) << dendl; - return r; - } else { - bool rollback = false; - BOOST_SCOPE_EXIT_ALL(ictx, &mirror_image_internal, &rollback) { - if (rollback) { - CephContext *cct = ictx->cct; - mirror_image_internal.state = cls::rbd::MIRROR_IMAGE_STATE_ENABLED; - int r = cls_client::mirror_image_set(&ictx->md_ctx, ictx->id, mirror_image_internal); - if (r < 0) { - lderr(cct) << "failed to re-enable image mirroring: " << cpp_strerror(r) - << dendl; - } - } - }; - - { - RWLock::RLocker l(ictx->snap_lock); - map snap_info = ictx->snap_info; - for (auto &info : snap_info) { - ParentSpec parent_spec(ictx->md_ctx.get_id(), ictx->id, info.first); - map< pair, set > image_info; - - r = list_children_info(ictx, parent_spec, image_info); - if (r < 0) { - rollback = true; - return r; - } - if (image_info.empty()) - continue; - - Rados rados(ictx->md_ctx); - for (auto &info: image_info) { - IoCtx ioctx; - r = rados.ioctx_create2(info.first.first, ioctx); - if (r < 0) { - rollback = true; - lderr(cct) << "Error accessing child image pool " << info.first.second << dendl; - return r; - } - for (auto &id_it : info.second) { - cls::rbd::MirrorImage mirror_image_internal; - r = cls_client::mirror_image_get(&ioctx, id_it, &mirror_image_internal); - if (r != -ENOENT) { - rollback = true; - lderr(cct) << "mirroring is enabled on one or more children " << dendl; - return -EBUSY; - } - } - } - } - } - - r = mirror_image_disable_internal(ictx, force); - if (r < 0) { - rollback = true; - return r; - } - } - - return 0; - } - - int mirror_image_promote(ImageCtx *ictx, bool force) { - CephContext *cct = ictx->cct; - ldout(cct, 20) << __func__ << ": ictx=" << ictx << ", " - << "force=" << force << dendl; - - int r = ictx->state->refresh_if_required(); - if (r < 0) { - return r; - } - - r = validate_mirroring_enabled(ictx); - if (r < 0) { - return r; - } - - std::string mirror_uuid; - r = Journal<>::get_tag_owner(ictx, &mirror_uuid); - if (r < 0) { - lderr(cct) << "failed to determine tag ownership: " << cpp_strerror(r) - << dendl; - return r; - } else if (mirror_uuid == Journal<>::LOCAL_MIRROR_UUID) { - lderr(cct) << "image is already primary" << dendl; - return -EINVAL; - } else if (mirror_uuid != Journal<>::ORPHAN_MIRROR_UUID && !force) { - lderr(cct) << "image is still primary within a remote cluster" << dendl; - return -EBUSY; - } - - // TODO: need interlock with local rbd-mirror daemon to ensure it has stopped - // replay - - r = Journal<>::promote(ictx); - if (r < 0) { - lderr(cct) << "failed to promote image: " << cpp_strerror(r) - << dendl; - return r; - } - return 0; - } - - int mirror_image_demote(ImageCtx *ictx) { - CephContext *cct = ictx->cct; - ldout(cct, 20) << __func__ << ": ictx=" << ictx << dendl; - - int r = ictx->state->refresh_if_required(); - if (r < 0) { - return r; - } - - r = validate_mirroring_enabled(ictx); - if (r < 0) { - return r; - } - - bool is_primary; - r = Journal<>::is_tag_owner(ictx, &is_primary); - if (r < 0) { - lderr(cct) << "failed to determine tag ownership: " << cpp_strerror(r) - << dendl; - return r; - } - - if (!is_primary) { - lderr(cct) << "image is not currently the primary" << dendl; - return -EINVAL; - } - - RWLock::RLocker owner_lock(ictx->owner_lock); - if (ictx->exclusive_lock == nullptr) { - lderr(cct) << "exclusive lock is not active" << dendl; - return -EINVAL; - } - - // avoid accepting new requests from peers while we demote - // the image - ictx->exclusive_lock->block_requests(0); - BOOST_SCOPE_EXIT_ALL( (ictx) ) { - if (ictx->exclusive_lock != nullptr) { - ictx->exclusive_lock->unblock_requests(); - } - }; - - C_SaferCond lock_ctx; - ictx->exclusive_lock->acquire_lock(&lock_ctx); - - // don't block holding lock since refresh might be required - ictx->owner_lock.put_read(); - r = lock_ctx.wait(); - ictx->owner_lock.get_read(); - - if (r < 0) { - lderr(cct) << "failed to lock image: " << cpp_strerror(r) << dendl; - return r; - } else if (ictx->exclusive_lock == nullptr || - !ictx->exclusive_lock->is_lock_owner()) { - lderr(cct) << "failed to acquire exclusive lock" << dendl; - return -EROFS; - } - - BOOST_SCOPE_EXIT_ALL( (ictx) ) { - C_SaferCond lock_ctx; - ictx->exclusive_lock->release_lock(&lock_ctx); - lock_ctx.wait(); - }; - - RWLock::RLocker snap_locker(ictx->snap_lock); - if (ictx->journal == nullptr) { - lderr(cct) << "journal is not active" << dendl; - return -EINVAL; - } else if (!ictx->journal->is_tag_owner()) { - lderr(cct) << "image is not currently the primary" << dendl; - return -EINVAL; - } - - r = ictx->journal->demote(); - if (r < 0) { - lderr(cct) << "failed to demote image: " << cpp_strerror(r) - << dendl; - return r; - } - return 0; - } - - int mirror_image_resync(ImageCtx *ictx) { - CephContext *cct = ictx->cct; - ldout(cct, 20) << __func__ << ": ictx=" << ictx << dendl; - - int r = ictx->state->refresh_if_required(); - if (r < 0) { - return r; - } - - r = validate_mirroring_enabled(ictx); - if (r < 0) { - return r; - } - - std::string mirror_uuid; - r = Journal<>::get_tag_owner(ictx, &mirror_uuid); - if (r < 0) { - lderr(cct) << "failed to determine tag ownership: " << cpp_strerror(r) - << dendl; - return r; - } else if (mirror_uuid == Journal<>::LOCAL_MIRROR_UUID) { - lderr(cct) << "image is primary, cannot resync to itself" << dendl; - return -EINVAL; - } - - // flag the journal indicating that we want to rebuild the local image - r = Journal<>::request_resync(ictx); - if (r < 0) { - lderr(cct) << "failed to request resync: " << cpp_strerror(r) << dendl; - return r; - } - - return 0; - } - - int mirror_image_get_info(ImageCtx *ictx, mirror_image_info_t *mirror_image_info, - size_t info_size) { - CephContext *cct = ictx->cct; - ldout(cct, 20) << __func__ << ": ictx=" << ictx << dendl; - if (info_size < sizeof(mirror_image_info_t)) { - return -ERANGE; - } - - int r = ictx->state->refresh_if_required(); - if (r < 0) { - return r; - } - - cls::rbd::MirrorImage mirror_image_internal; - r = cls_client::mirror_image_get(&ictx->md_ctx, ictx->id, - &mirror_image_internal); - if (r < 0 && r != -ENOENT) { - lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) - << dendl; - return r; - } - - mirror_image_info->global_id = mirror_image_internal.global_image_id; - if (r == -ENOENT) { - mirror_image_info->state = RBD_MIRROR_IMAGE_DISABLED; - } else { - mirror_image_info->state = - static_cast(mirror_image_internal.state); - } - - if (mirror_image_info->state == RBD_MIRROR_IMAGE_ENABLED) { - r = Journal<>::is_tag_owner(ictx, &mirror_image_info->primary); - if (r < 0) { - lderr(cct) << "failed to check tag ownership: " - << cpp_strerror(r) << dendl; - return r; - } - } else { - mirror_image_info->primary = false; - } - - return 0; - } - - int mirror_image_get_status(ImageCtx *ictx, mirror_image_status_t *status, - size_t status_size) { - CephContext *cct = ictx->cct; - ldout(cct, 20) << __func__ << ": ictx=" << ictx << dendl; - if (status_size < sizeof(mirror_image_status_t)) { - return -ERANGE; - } - - int r = ictx->state->refresh_if_required(); - if (r < 0) { - return r; - } - - mirror_image_info_t info; - r = mirror_image_get_info(ictx, &info, sizeof(info)); - if (r < 0) { - return r; - } - - cls::rbd::MirrorImageStatus - s(cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN, "status not found"); - - r = cls_client::mirror_image_status_get(&ictx->md_ctx, info.global_id, &s); - if (r < 0 && r != -ENOENT) { - lderr(cct) << "failed to retrieve image mirror status: " - << cpp_strerror(r) << dendl; - return r; - } - - *status = mirror_image_status_t{ - ictx->name, - info, - static_cast(s.state), - s.description, - s.last_update.sec(), - s.up}; - return 0; - } - - int mirror_mode_get(IoCtx& io_ctx, rbd_mirror_mode_t *mirror_mode) { - CephContext *cct = reinterpret_cast(io_ctx.cct()); - ldout(cct, 20) << __func__ << dendl; - - cls::rbd::MirrorMode mirror_mode_internal; - int r = cls_client::mirror_mode_get(&io_ctx, &mirror_mode_internal); - if (r < 0) { - lderr(cct) << "Failed to retrieve mirror mode: " << cpp_strerror(r) - << dendl; - return r; - } - - switch (mirror_mode_internal) { - case cls::rbd::MIRROR_MODE_DISABLED: - case cls::rbd::MIRROR_MODE_IMAGE: - case cls::rbd::MIRROR_MODE_POOL: - *mirror_mode = static_cast(mirror_mode_internal); - break; - default: - lderr(cct) << "Unknown mirror mode (" - << static_cast(mirror_mode_internal) << ")" - << dendl; - return -EINVAL; - } - return 0; - } - - int list_mirror_images(IoCtx& io_ctx, - std::set& mirror_image_ids) { - CephContext *cct = reinterpret_cast(io_ctx.cct()); - - std::string last_read = ""; - int max_read = 1024; - int r; - do { - std::map mirror_images; - r = cls_client::mirror_image_list(&io_ctx, last_read, max_read, - &mirror_images); - if (r < 0) { - lderr(cct) << "error listing mirrored image directory: " - << cpp_strerror(r) << dendl; - return r; - } - for (auto it = mirror_images.begin(); it != mirror_images.end(); ++it) { - mirror_image_ids.insert(it->first); - } - if (!mirror_images.empty()) { - last_read = mirror_images.rbegin()->first; - } - r = mirror_images.size(); - } while (r == max_read); - - return 0; - } - - int mirror_mode_set(IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode) { - CephContext *cct = reinterpret_cast(io_ctx.cct()); - ldout(cct, 20) << __func__ << dendl; - - cls::rbd::MirrorMode next_mirror_mode; - switch (mirror_mode) { - case RBD_MIRROR_MODE_DISABLED: - case RBD_MIRROR_MODE_IMAGE: - case RBD_MIRROR_MODE_POOL: - next_mirror_mode = static_cast(mirror_mode); - break; - default: - lderr(cct) << "Unknown mirror mode (" - << static_cast(mirror_mode) << ")" << dendl; - return -EINVAL; - } - - int r; - if (next_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) { - // fail early if pool still has peers registered and attempting to disable - std::vector mirror_peers; - r = cls_client::mirror_peer_list(&io_ctx, &mirror_peers); - if (r < 0 && r != -ENOENT) { - lderr(cct) << "Failed to list peers: " << cpp_strerror(r) << dendl; - return r; - } else if (!mirror_peers.empty()) { - lderr(cct) << "mirror peers still registered" << dendl; - return -EBUSY; - } - } - - cls::rbd::MirrorMode current_mirror_mode; - r = cls_client::mirror_mode_get(&io_ctx, ¤t_mirror_mode); - if (r < 0) { - lderr(cct) << "Failed to retrieve mirror mode: " << cpp_strerror(r) - << dendl; - return r; - } - - if (current_mirror_mode == next_mirror_mode) { - return 0; - } else if (current_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) { - uuid_d uuid_gen; - uuid_gen.generate_random(); - r = cls_client::mirror_uuid_set(&io_ctx, uuid_gen.to_string()); - if (r < 0) { - lderr(cct) << "Failed to allocate mirroring uuid: " << cpp_strerror(r) - << dendl; - return r; - } - } - - if (current_mirror_mode != cls::rbd::MIRROR_MODE_IMAGE) { - r = cls_client::mirror_mode_set(&io_ctx, cls::rbd::MIRROR_MODE_IMAGE); - if (r < 0) { - lderr(cct) << "failed to set mirror mode to image: " - << cpp_strerror(r) << dendl; - return r; - } - - r = MirroringWatcher<>::notify_mode_updated(io_ctx, - cls::rbd::MIRROR_MODE_IMAGE); - if (r < 0) { - lderr(cct) << "failed to send update notification: " << cpp_strerror(r) - << dendl; - } - } - - if (next_mirror_mode == cls::rbd::MIRROR_MODE_IMAGE) { - return 0; - } - - if (next_mirror_mode == cls::rbd::MIRROR_MODE_POOL) { - map images; - r = list_images_v2(io_ctx, images); - if (r < 0) { - lderr(cct) << "Failed listing images: " << cpp_strerror(r) << dendl; - return r; - } - - for (const auto& img_pair : images) { - uint64_t features; - r = cls_client::get_features(&io_ctx, - util::header_name(img_pair.second), - CEPH_NOSNAP, &features); - if (r < 0) { - lderr(cct) << "error getting features for image " << img_pair.first - << ": " << cpp_strerror(r) << dendl; - return r; - } - - if ((features & RBD_FEATURE_JOURNALING) != 0) { - ImageCtx *img_ctx = new ImageCtx("", img_pair.second, nullptr, - io_ctx, false); - r = img_ctx->state->open(false); - if (r < 0) { - lderr(cct) << "error opening image "<< img_pair.first << ": " - << cpp_strerror(r) << dendl; - return r; - } - - r = mirror_image_enable(img_ctx, true); - if (r < 0) { - lderr(cct) << "error enabling mirroring for image " - << img_pair.first << ": " << cpp_strerror(r) << dendl; - img_ctx->state->close(); - return r; - } - - r = img_ctx->state->close(); - if (r < 0) { - lderr(cct) << "failed to close image " << img_pair.first << ": " - << cpp_strerror(r) << dendl; - return r; - } - } - } - } else if (next_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) { - std::set image_ids; - r = list_mirror_images(io_ctx, image_ids); - if (r < 0) { - lderr(cct) << "Failed listing images: " << cpp_strerror(r) << dendl; - return r; - } - - for (const auto& img_id : image_ids) { - if (current_mirror_mode == cls::rbd::MIRROR_MODE_IMAGE) { - cls::rbd::MirrorImage mirror_image; - r = cls_client::mirror_image_get(&io_ctx, img_id, &mirror_image); - if (r < 0 && r != -ENOENT) { - lderr(cct) << "failed to retrieve mirroring state for image id " - << img_id << ": " << cpp_strerror(r) << dendl; - return r; - } - if (mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { - lderr(cct) << "Failed to disable mirror mode: there are still " - << "images with mirroring enabled" << dendl; - return -EINVAL; - } - } else { - ImageCtx *img_ctx = new ImageCtx("", img_id, nullptr, io_ctx, false); - r = img_ctx->state->open(false); - if (r < 0) { - lderr(cct) << "error opening image id "<< img_id << ": " - << cpp_strerror(r) << dendl; - return r; - } - - r = mirror_image_disable(img_ctx, false); - int close_r = img_ctx->state->close(); - if (r < 0) { - lderr(cct) << "error disabling mirroring for image id " << img_id - << cpp_strerror(r) << dendl; - return r; - } else if (close_r < 0) { - lderr(cct) << "failed to close image id " << img_id << ": " - << cpp_strerror(close_r) << dendl; - return close_r; - } - } - } - } - - r = cls_client::mirror_mode_set(&io_ctx, next_mirror_mode); - if (r < 0) { - lderr(cct) << "Failed to set mirror mode: " << cpp_strerror(r) << dendl; - return r; - } - - r = MirroringWatcher<>::notify_mode_updated(io_ctx, next_mirror_mode); - if (r < 0) { - lderr(cct) << "failed to send update notification: " << cpp_strerror(r) - << dendl; - } - return 0; - } - - int mirror_peer_add(IoCtx& io_ctx, std::string *uuid, - const std::string &cluster_name, - const std::string &client_name) { - CephContext *cct = reinterpret_cast(io_ctx.cct()); - ldout(cct, 20) << __func__ << ": " - << "name=" << cluster_name << ", " - << "client=" << client_name << dendl; - - if (cct->_conf->cluster == cluster_name) { - lderr(cct) << "Cannot add self as remote peer" << dendl; - return -EINVAL; - } - - int r; - do { - uuid_d uuid_gen; - uuid_gen.generate_random(); - - *uuid = uuid_gen.to_string(); - r = cls_client::mirror_peer_add(&io_ctx, *uuid, cluster_name, - client_name); - if (r == -ESTALE) { - ldout(cct, 5) << "Duplicate UUID detected, retrying" << dendl; - } else if (r < 0) { - lderr(cct) << "Failed to add mirror peer '" << uuid << "': " - << cpp_strerror(r) << dendl; - return r; - } - } while (r == -ESTALE); - return 0; - } - - int mirror_peer_remove(IoCtx& io_ctx, const std::string &uuid) { - CephContext *cct = reinterpret_cast(io_ctx.cct()); - ldout(cct, 20) << __func__ << ": uuid=" << uuid << dendl; - - int r = cls_client::mirror_peer_remove(&io_ctx, uuid); - if (r < 0 && r != -ENOENT) { - lderr(cct) << "Failed to remove peer '" << uuid << "': " - << cpp_strerror(r) << dendl; - return r; - } - return 0; - } - - int mirror_peer_list(IoCtx& io_ctx, std::vector *peers) { - CephContext *cct = reinterpret_cast(io_ctx.cct()); - ldout(cct, 20) << __func__ << dendl; - - std::vector mirror_peers; - int r = cls_client::mirror_peer_list(&io_ctx, &mirror_peers); - if (r < 0 && r != -ENOENT) { - lderr(cct) << "Failed to list peers: " << cpp_strerror(r) << dendl; - return r; - } - - peers->clear(); - peers->reserve(mirror_peers.size()); - for (auto &mirror_peer : mirror_peers) { - mirror_peer_t peer; - peer.uuid = mirror_peer.uuid; - peer.cluster_name = mirror_peer.cluster_name; - peer.client_name = mirror_peer.client_name; - peers->push_back(peer); - } - return 0; - } - - int mirror_peer_set_client(IoCtx& io_ctx, const std::string &uuid, - const std::string &client_name) { - CephContext *cct = reinterpret_cast(io_ctx.cct()); - ldout(cct, 20) << __func__ << ": uuid=" << uuid << ", " - << "client=" << client_name << dendl; - - int r = cls_client::mirror_peer_set_client(&io_ctx, uuid, client_name); - if (r < 0) { - lderr(cct) << "Failed to update client '" << uuid << "': " - << cpp_strerror(r) << dendl; - return r; - } - return 0; - } - - int mirror_peer_set_cluster(IoCtx& io_ctx, const std::string &uuid, - const std::string &cluster_name) { - CephContext *cct = reinterpret_cast(io_ctx.cct()); - ldout(cct, 20) << __func__ << ": uuid=" << uuid << ", " - << "cluster=" << cluster_name << dendl; - - int r = cls_client::mirror_peer_set_cluster(&io_ctx, uuid, cluster_name); - if (r < 0) { - lderr(cct) << "Failed to update cluster '" << uuid << "': " - << cpp_strerror(r) << dendl; - return r; - } - return 0; - } - - int mirror_image_status_list(IoCtx& io_ctx, const std::string &start_id, - size_t max, std::map *images) { - CephContext *cct = reinterpret_cast(io_ctx.cct()); - int r; - - map id_to_name; - { - map name_to_id; - r = list_images_v2(io_ctx, name_to_id); - if (r < 0) { - return r; - } - for (auto it : name_to_id) { - id_to_name[it.second] = it.first; - } - } - - map images_; - map statuses_; - - r = librbd::cls_client::mirror_image_status_list(&io_ctx, start_id, max, - &images_, &statuses_); - if (r < 0) { - lderr(cct) << "Failed to list mirror image statuses: " - << cpp_strerror(r) << dendl; - return r; - } - - cls::rbd::MirrorImageStatus unknown_status( - cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN, "status not found"); - - for (auto it = images_.begin(); it != images_.end(); ++it) { - auto &image_id = it->first; - auto &info = it->second; - auto &image_name = id_to_name[image_id]; - if (image_name.empty()) { - lderr(cct) << "Failed to find image name for image " << image_id - << ", using image id as name" << dendl; - image_name = image_id; - } - auto s_it = statuses_.find(image_id); - auto &s = s_it != statuses_.end() ? s_it->second : unknown_status; - (*images)[image_id] = mirror_image_status_t{ - image_name, - mirror_image_info_t{ - info.global_image_id, - static_cast(info.state), - false}, // XXX: To set "primary" right would require an additional call. - static_cast(s.state), - s.description, - s.last_update.sec(), - s.up}; - } - - return 0; - } - - int mirror_image_status_summary(IoCtx& io_ctx, - std::map *states) { - CephContext *cct = reinterpret_cast(io_ctx.cct()); - - std::map states_; - int r = cls_client::mirror_image_status_get_summary(&io_ctx, &states_); - if (r < 0) { - lderr(cct) << "Failed to get mirror status summary: " - << cpp_strerror(r) << dendl; - return r; - } - for (auto &s : states_) { - (*states)[static_cast(s.first)] = s.second; - } - return 0; - } - struct C_RBD_Readahead : public Context { ImageCtx *ictx; object_t oid; diff --git a/src/librbd/internal.h b/src/librbd/internal.h index 28a3c387528..80043f2d5f1 100644 --- a/src/librbd/internal.h +++ b/src/librbd/internal.h @@ -95,13 +95,9 @@ namespace librbd { int snap_set(ImageCtx *ictx, const char *snap_name); - int list_images_v2(librados::IoCtx& io_ctx, - std::map& images); int list(librados::IoCtx& io_ctx, std::vector& names); int list_children(ImageCtx *ictx, std::set > & names); - int list_children_info(ImageCtx *ictx, const ParentSpec &parent_spec, - std::map, std::set >& image_info); int create(librados::IoCtx& io_ctx, const char *imgname, uint64_t size, int *order); int create(librados::IoCtx& io_ctx, const char *imgname, uint64_t size, @@ -202,32 +198,6 @@ namespace librbd { int metadata_list(ImageCtx *ictx, const string &last, uint64_t max, map *pairs); int metadata_get(ImageCtx *ictx, const std::string &key, std::string *value); - int mirror_mode_get(IoCtx& io_ctx, rbd_mirror_mode_t *mirror_mode); - int mirror_mode_set(IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode); - int mirror_peer_add(IoCtx& io_ctx, std::string *uuid, - const std::string &cluster_name, - const std::string &client_name); - int mirror_peer_remove(IoCtx& io_ctx, const std::string &uuid); - int mirror_peer_list(IoCtx& io_ctx, std::vector *peers); - int mirror_peer_set_client(IoCtx& io_ctx, const std::string &uuid, - const std::string &client_name); - int mirror_peer_set_cluster(IoCtx& io_ctx, const std::string &uuid, - const std::string &cluster_name); - int mirror_image_status_list(IoCtx& io_ctx, const std::string &start_id, - size_t max, std::map *images); - int mirror_image_status_summary(IoCtx& io_ctx, - std::map *states); - - int mirror_image_enable(ImageCtx *ictx, bool relax_same_pool_parent_check); - int mirror_image_disable(ImageCtx *ictx, bool force); - int mirror_image_promote(ImageCtx *ictx, bool force); - int mirror_image_demote(ImageCtx *ictx); - int mirror_image_resync(ImageCtx *ictx); - int mirror_image_get_info(ImageCtx *ictx, mirror_image_info_t *mirror_image_info, - size_t info_size); - int mirror_image_get_status(ImageCtx *ictx, mirror_image_status_t *status, - size_t status_size); - } #endif diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index 584b6e994a3..390af617d50 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -1,4 +1,4 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab /* * Ceph - scalable distributed file system @@ -28,6 +28,7 @@ #include "librbd/ImageState.h" #include "librbd/internal.h" #include "librbd/Operations.h" +#include "librbd/api/Mirror.h" #include "librbd/io/AioCompletion.h" #include "librbd/io/ImageRequestWQ.h" #include "librbd/io/ReadResult.h" @@ -428,45 +429,47 @@ namespace librbd { } int RBD::mirror_mode_get(IoCtx& io_ctx, rbd_mirror_mode_t *mirror_mode) { - return librbd::mirror_mode_get(io_ctx, mirror_mode); + return librbd::api::Mirror<>::mode_get(io_ctx, mirror_mode); } int RBD::mirror_mode_set(IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode) { - return librbd::mirror_mode_set(io_ctx, mirror_mode); + return librbd::api::Mirror<>::mode_set(io_ctx, mirror_mode); } int RBD::mirror_peer_add(IoCtx& io_ctx, std::string *uuid, const std::string &cluster_name, const std::string &client_name) { - return librbd::mirror_peer_add(io_ctx, uuid, cluster_name, client_name); + return librbd::api::Mirror<>::peer_add(io_ctx, uuid, cluster_name, + client_name); } int RBD::mirror_peer_remove(IoCtx& io_ctx, const std::string &uuid) { - return librbd::mirror_peer_remove(io_ctx, uuid); + return librbd::api::Mirror<>::peer_remove(io_ctx, uuid); } int RBD::mirror_peer_list(IoCtx& io_ctx, std::vector *peers) { - return librbd::mirror_peer_list(io_ctx, peers); + return librbd::api::Mirror<>::peer_list(io_ctx, peers); } int RBD::mirror_peer_set_client(IoCtx& io_ctx, const std::string &uuid, const std::string &client_name) { - return librbd::mirror_peer_set_client(io_ctx, uuid, client_name); + return librbd::api::Mirror<>::peer_set_client(io_ctx, uuid, client_name); } int RBD::mirror_peer_set_cluster(IoCtx& io_ctx, const std::string &uuid, const std::string &cluster_name) { - return librbd::mirror_peer_set_cluster(io_ctx, uuid, cluster_name); + return librbd::api::Mirror<>::peer_set_cluster(io_ctx, uuid, cluster_name); } int RBD::mirror_image_status_list(IoCtx& io_ctx, const std::string &start_id, size_t max, std::map *images) { - return librbd::mirror_image_status_list(io_ctx, start_id, max, images); + return librbd::api::Mirror<>::image_status_list(io_ctx, start_id, max, + images); } int RBD::mirror_image_status_summary(IoCtx& io_ctx, std::map *states) { - return librbd::mirror_image_status_summary(io_ctx, states); + return librbd::api::Mirror<>::image_status_summary(io_ctx, states); } int RBD::group_create(IoCtx& io_ctx, const char *group_name) @@ -1537,41 +1540,42 @@ namespace librbd { int Image::mirror_image_enable() { ImageCtx *ictx = (ImageCtx *)ctx; - return librbd::mirror_image_enable(ictx, false); + return librbd::api::Mirror<>::image_enable(ictx, false); } int Image::mirror_image_disable(bool force) { ImageCtx *ictx = (ImageCtx *)ctx; - return librbd::mirror_image_disable(ictx, force); + return librbd::api::Mirror<>::image_disable(ictx, force); } int Image::mirror_image_promote(bool force) { ImageCtx *ictx = (ImageCtx *)ctx; - return librbd::mirror_image_promote(ictx, force); + return librbd::api::Mirror<>::image_promote(ictx, force); } int Image::mirror_image_demote() { ImageCtx *ictx = (ImageCtx *)ctx; - return librbd::mirror_image_demote(ictx); + return librbd::api::Mirror<>::image_demote(ictx); } int Image::mirror_image_resync() { ImageCtx *ictx = (ImageCtx *)ctx; - return librbd::mirror_image_resync(ictx); + return librbd::api::Mirror<>::image_resync(ictx); } int Image::mirror_image_get_info(mirror_image_info_t *mirror_image_info, size_t info_size) { ImageCtx *ictx = (ImageCtx *)ctx; - return librbd::mirror_image_get_info(ictx, mirror_image_info, info_size); + return librbd::api::Mirror<>::image_get_info(ictx, mirror_image_info, + info_size); } int Image::mirror_image_get_status(mirror_image_status_t *mirror_image_status, size_t status_size) { ImageCtx *ictx = (ImageCtx *)ctx; - return librbd::mirror_image_get_status(ictx, mirror_image_status, - status_size); + return librbd::api::Mirror<>::image_get_status(ictx, mirror_image_status, + status_size); } int Image::update_watch(UpdateWatchCtx *wctx, uint64_t *handle) { @@ -1676,14 +1680,14 @@ extern "C" int rbd_mirror_mode_get(rados_ioctx_t p, rbd_mirror_mode_t *mirror_mode) { librados::IoCtx io_ctx; librados::IoCtx::from_rados_ioctx_t(p, io_ctx); - return librbd::mirror_mode_get(io_ctx, mirror_mode); + return librbd::api::Mirror<>::mode_get(io_ctx, mirror_mode); } extern "C" int rbd_mirror_mode_set(rados_ioctx_t p, rbd_mirror_mode_t mirror_mode) { librados::IoCtx io_ctx; librados::IoCtx::from_rados_ioctx_t(p, io_ctx); - return librbd::mirror_mode_set(io_ctx, mirror_mode); + return librbd::api::Mirror<>::mode_set(io_ctx, mirror_mode); } extern "C" int rbd_mirror_peer_add(rados_ioctx_t p, char *uuid, @@ -1700,7 +1704,8 @@ extern "C" int rbd_mirror_peer_add(rados_ioctx_t p, char *uuid, } std::string uuid_str; - int r = librbd::mirror_peer_add(io_ctx, &uuid_str, cluster_name, client_name); + int r = librbd::api::Mirror<>::peer_add(io_ctx, &uuid_str, cluster_name, + client_name); if (r >= 0) { strncpy(uuid, uuid_str.c_str(), uuid_max_length); uuid[uuid_max_length - 1] = '\0'; @@ -1711,7 +1716,7 @@ extern "C" int rbd_mirror_peer_add(rados_ioctx_t p, char *uuid, extern "C" int rbd_mirror_peer_remove(rados_ioctx_t p, const char *uuid) { librados::IoCtx io_ctx; librados::IoCtx::from_rados_ioctx_t(p, io_ctx); - int r = librbd::mirror_peer_remove(io_ctx, uuid); + int r = librbd::api::Mirror<>::peer_remove(io_ctx, uuid); return r; } @@ -1721,7 +1726,7 @@ extern "C" int rbd_mirror_peer_list(rados_ioctx_t p, librados::IoCtx::from_rados_ioctx_t(p, io_ctx); std::vector peer_vector; - int r = librbd::mirror_peer_list(io_ctx, &peer_vector); + int r = librbd::api::Mirror<>::peer_list(io_ctx, &peer_vector); if (r < 0) { return r; } @@ -1753,14 +1758,14 @@ extern "C" int rbd_mirror_peer_set_client(rados_ioctx_t p, const char *uuid, const char *client_name) { librados::IoCtx io_ctx; librados::IoCtx::from_rados_ioctx_t(p, io_ctx); - return librbd::mirror_peer_set_client(io_ctx, uuid, client_name); + return librbd::api::Mirror<>::peer_set_client(io_ctx, uuid, client_name); } extern "C" int rbd_mirror_peer_set_cluster(rados_ioctx_t p, const char *uuid, const char *cluster_name) { librados::IoCtx io_ctx; librados::IoCtx::from_rados_ioctx_t(p, io_ctx); - return librbd::mirror_peer_set_cluster(io_ctx, uuid, cluster_name); + return librbd::api::Mirror<>::peer_set_cluster(io_ctx, uuid, cluster_name); } extern "C" int rbd_mirror_image_status_list(rados_ioctx_t p, @@ -1770,7 +1775,8 @@ extern "C" int rbd_mirror_image_status_list(rados_ioctx_t p, librados::IoCtx::from_rados_ioctx_t(p, io_ctx); std::map cpp_images; - int r = librbd::mirror_image_status_list(io_ctx, start_id, max, &cpp_images); + int r = librbd::api::Mirror<>::image_status_list(io_ctx, start_id, max, + &cpp_images); if (r < 0) { return r; } @@ -1804,7 +1810,7 @@ extern "C" int rbd_mirror_image_status_summary(rados_ioctx_t p, librados::IoCtx::from_rados_ioctx_t(p, io_ctx); std::map states_; - int r = librbd::mirror_image_status_summary(io_ctx, &states_); + int r = librbd::api::Mirror<>::image_status_summary(io_ctx, &states_); if (r < 0) { return r; } @@ -3268,31 +3274,31 @@ extern "C" int rbd_metadata_list(rbd_image_t image, const char *start, uint64_t extern "C" int rbd_mirror_image_enable(rbd_image_t image) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; - return librbd::mirror_image_enable(ictx, false); + return librbd::api::Mirror<>::image_enable(ictx, false); } extern "C" int rbd_mirror_image_disable(rbd_image_t image, bool force) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; - return librbd::mirror_image_disable(ictx, force); + return librbd::api::Mirror<>::image_disable(ictx, force); } extern "C" int rbd_mirror_image_promote(rbd_image_t image, bool force) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; - return librbd::mirror_image_promote(ictx, force); + return librbd::api::Mirror<>::image_promote(ictx, force); } extern "C" int rbd_mirror_image_demote(rbd_image_t image) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; - return librbd::mirror_image_demote(ictx); + return librbd::api::Mirror<>::image_demote(ictx); } extern "C" int rbd_mirror_image_resync(rbd_image_t image) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; - return librbd::mirror_image_resync(ictx); + return librbd::api::Mirror<>::image_resync(ictx); } extern "C" int rbd_mirror_image_get_info(rbd_image_t image, @@ -3302,8 +3308,8 @@ extern "C" int rbd_mirror_image_get_info(rbd_image_t image, librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; librbd::mirror_image_info_t cpp_mirror_image; - int r = librbd::mirror_image_get_info(ictx, &cpp_mirror_image, - sizeof(cpp_mirror_image)); + int r = librbd::api::Mirror<>::image_get_info(ictx, &cpp_mirror_image, + sizeof(cpp_mirror_image)); if (r < 0) { return r; } @@ -3319,8 +3325,8 @@ extern "C" int rbd_mirror_image_get_status(rbd_image_t image, librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; librbd::mirror_image_status_t cpp_status; - int r = librbd::mirror_image_get_status(ictx, &cpp_status, - sizeof(cpp_status)); + int r = librbd::api::Mirror<>::image_get_status(ictx, &cpp_status, + sizeof(cpp_status)); if (r < 0) { return r; } diff --git a/src/test/rbd_mirror/test_ClusterWatcher.cc b/src/test/rbd_mirror/test_ClusterWatcher.cc index 43da46f4ce6..a0c39876d9c 100644 --- a/src/test/rbd_mirror/test_ClusterWatcher.cc +++ b/src/test/rbd_mirror/test_ClusterWatcher.cc @@ -5,6 +5,7 @@ #include "common/errno.h" #include "common/Mutex.h" #include "librbd/internal.h" +#include "librbd/api/Mirror.h" #include "tools/rbd_mirror/ClusterWatcher.h" #include "tools/rbd_mirror/types.h" #include "test/rbd_mirror/test_fixture.h" @@ -54,13 +55,15 @@ public: if (enable_mirroring) { librados::IoCtx ioctx; ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx)); - ASSERT_EQ(0, librbd::mirror_mode_set(ioctx, RBD_MIRROR_MODE_POOL)); + ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx, + RBD_MIRROR_MODE_POOL)); std::string gen_uuid; - ASSERT_EQ(0, librbd::mirror_peer_add(ioctx, - uuid != nullptr ? uuid : &gen_uuid, - peer.cluster_name, - peer.client_name)); + ASSERT_EQ(0, librbd::api::Mirror<>::peer_add(ioctx, + uuid != nullptr ? uuid : + &gen_uuid, + peer.cluster_name, + peer.client_name)); m_pool_peers[pool_id].insert(peer); m_mirrored_pools.insert(pool_name); } diff --git a/src/test/rbd_mirror/test_ImageDeleter.cc b/src/test/rbd_mirror/test_ImageDeleter.cc index c61aaaaf70e..1059e20b7c2 100644 --- a/src/test/rbd_mirror/test_ImageDeleter.cc +++ b/src/test/rbd_mirror/test_ImageDeleter.cc @@ -25,6 +25,7 @@ #include "librbd/Journal.h" #include "librbd/internal.h" #include "librbd/Utils.h" +#include "librbd/api/Mirror.h" #include "test/rbd_mirror/test_fixture.h" #include "test/librados/test.h" @@ -61,7 +62,7 @@ public: void SetUp() override { TestFixture::SetUp(); - librbd::mirror_mode_set(m_local_io_ctx, RBD_MIRROR_MODE_IMAGE); + librbd::api::Mirror<>::mode_set(m_local_io_ctx, RBD_MIRROR_MODE_IMAGE); m_deleter = new rbd::mirror::ImageDeleter(m_threads->work_queue, m_threads->timer, @@ -120,7 +121,7 @@ public: EXPECT_EQ(1, r == 0 || r == -ENOENT); if (r == 0) { - int r2 = librbd::mirror_image_promote(ictx, true); + int r2 = librbd::api::Mirror<>::image_promote(ictx, true); EXPECT_EQ(1, r2 == 0 || r2 == -EINVAL); } @@ -138,7 +139,7 @@ public: close = true; } - EXPECT_EQ(0, librbd::mirror_image_demote(ictx)); + EXPECT_EQ(0, librbd::api::Mirror<>::image_demote(ictx)); if (close) { EXPECT_EQ(0, ictx->state->close()); diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index 554d044c1ab..35139384149 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -30,6 +30,7 @@ #include "librbd/Operations.h" #include "librbd/Utils.h" #include "librbd/internal.h" +#include "librbd/api/Mirror.h" #include "librbd/io/AioCompletion.h" #include "librbd/io/ImageRequestWQ.h" #include "librbd/io/ReadResult.h" @@ -100,7 +101,8 @@ public: EXPECT_EQ(0, m_remote_cluster.ioctx_create(m_remote_pool_name.c_str(), m_remote_ioctx)); - EXPECT_EQ(0, librbd::mirror_mode_set(m_remote_ioctx, RBD_MIRROR_MODE_POOL)); + EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, + RBD_MIRROR_MODE_POOL)); m_image_name = get_temp_image_name(); uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context); @@ -415,10 +417,11 @@ TEST_F(TestImageReplayer, BootstrapErrorNoJournal) TEST_F(TestImageReplayer, BootstrapErrorMirrorDisabled) { // disable remote image mirroring - ASSERT_EQ(0, librbd::mirror_mode_set(m_remote_ioctx, RBD_MIRROR_MODE_IMAGE)); + ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, + RBD_MIRROR_MODE_IMAGE)); librbd::ImageCtx *ictx; open_remote_image(&ictx); - ASSERT_EQ(0, librbd::mirror_image_disable(ictx, true)); + ASSERT_EQ(0, librbd::api::Mirror<>::image_disable(ictx, true)); close_image(ictx); create_replayer<>(); @@ -430,10 +433,11 @@ TEST_F(TestImageReplayer, BootstrapErrorMirrorDisabled) TEST_F(TestImageReplayer, BootstrapMirrorDisabling) { // set remote image mirroring state to DISABLING - ASSERT_EQ(0, librbd::mirror_mode_set(m_remote_ioctx, RBD_MIRROR_MODE_IMAGE)); + ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, + RBD_MIRROR_MODE_IMAGE)); librbd::ImageCtx *ictx; open_remote_image(&ictx); - ASSERT_EQ(0, librbd::mirror_image_enable(ictx, false)); + ASSERT_EQ(0, librbd::api::Mirror<>::image_enable(ictx, false)); cls::rbd::MirrorImage mirror_image; ASSERT_EQ(0, librbd::cls_client::mirror_image_get(&m_remote_ioctx, ictx->id, &mirror_image)); @@ -454,7 +458,7 @@ TEST_F(TestImageReplayer, BootstrapDemoted) // demote remote image librbd::ImageCtx *ictx; open_remote_image(&ictx); - ASSERT_EQ(0, librbd::mirror_image_demote(ictx)); + ASSERT_EQ(0, librbd::api::Mirror<>::image_demote(ictx)); close_image(ictx); create_replayer<>(); diff --git a/src/test/rbd_mirror/test_LeaderWatcher.cc b/src/test/rbd_mirror/test_LeaderWatcher.cc index af7a92bd0ab..cb4b11a6438 100644 --- a/src/test/rbd_mirror/test_LeaderWatcher.cc +++ b/src/test/rbd_mirror/test_LeaderWatcher.cc @@ -4,6 +4,7 @@ #include "include/rados/librados.hpp" #include "librbd/internal.h" #include "librbd/Utils.h" +#include "librbd/api/Mirror.h" #include "test/librbd/test_support.h" #include "test/rbd_mirror/test_fixture.h" #include "tools/rbd_mirror/LeaderWatcher.h" @@ -89,7 +90,8 @@ public: void SetUp() override { TestFixture::SetUp(); - EXPECT_EQ(0, librbd::mirror_mode_set(m_local_io_ctx, RBD_MIRROR_MODE_POOL)); + EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_local_io_ctx, + RBD_MIRROR_MODE_POOL)); if (is_librados_test_stub()) { // speed testing up a little diff --git a/src/test/rbd_mirror/test_PoolWatcher.cc b/src/test/rbd_mirror/test_PoolWatcher.cc index 201934e3dca..bbc9bb123e0 100644 --- a/src/test/rbd_mirror/test_PoolWatcher.cc +++ b/src/test/rbd_mirror/test_PoolWatcher.cc @@ -12,6 +12,7 @@ #include "librbd/ImageState.h" #include "librbd/Operations.h" #include "librbd/Utils.h" +#include "librbd/api/Mirror.h" #include "common/Cond.h" #include "common/errno.h" #include "common/Mutex.h" @@ -68,11 +69,12 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"), m_pool_watcher.reset(new PoolWatcher(ioctx, 30, m_lock, m_cond)); if (enable_mirroring) { - ASSERT_EQ(0, librbd::mirror_mode_set(ioctx, RBD_MIRROR_MODE_POOL)); + ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx, + RBD_MIRROR_MODE_POOL)); std::string uuid; - ASSERT_EQ(0, librbd::mirror_peer_add(ioctx, &uuid, - peer.cluster_name, - peer.client_name)); + ASSERT_EQ(0, librbd::api::Mirror<>::peer_add(ioctx, &uuid, + peer.cluster_name, + peer.client_name)); } if (name != nullptr) { *name = pool_name; diff --git a/src/tools/rbd_mirror/ClusterWatcher.cc b/src/tools/rbd_mirror/ClusterWatcher.cc index 87b6f8ee52f..90dd61d405e 100644 --- a/src/tools/rbd_mirror/ClusterWatcher.cc +++ b/src/tools/rbd_mirror/ClusterWatcher.cc @@ -1,12 +1,11 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab +#include "ClusterWatcher.h" #include "common/debug.h" #include "common/errno.h" - #include "librbd/internal.h" - -#include "ClusterWatcher.h" +#include "librbd/api/Mirror.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rbd_mirror @@ -98,7 +97,7 @@ void ClusterWatcher::read_pool_peers(PoolPeers *pool_peers, } rbd_mirror_mode_t mirror_mode; - r = librbd::mirror_mode_get(ioctx, &mirror_mode); + r = librbd::api::Mirror<>::mode_get(ioctx, &mirror_mode); if (r < 0) { derr << "could not tell whether mirroring was enabled for " << pool_name << " : " << cpp_strerror(r) << dendl; @@ -110,7 +109,7 @@ void ClusterWatcher::read_pool_peers(PoolPeers *pool_peers, } vector configs; - r = librbd::mirror_peer_list(ioctx, &configs); + r = librbd::api::Mirror<>::peer_list(ioctx, &configs); if (r < 0) { derr << "error reading mirroring config for pool " << pool_name << cpp_strerror(r) << dendl; diff --git a/src/tools/rbd_mirror/PoolWatcher.cc b/src/tools/rbd_mirror/PoolWatcher.cc index 92c60699d56..f654855faab 100644 --- a/src/tools/rbd_mirror/PoolWatcher.cc +++ b/src/tools/rbd_mirror/PoolWatcher.cc @@ -9,7 +9,8 @@ #include "cls/rbd/cls_rbd_client.h" #include "include/rbd_types.h" #include "librbd/internal.h" - +#include "librbd/api/Image.h" +#include "librbd/api/Mirror.h" #include "PoolWatcher.h" #define dout_context g_ceph_context @@ -88,7 +89,7 @@ int PoolWatcher::refresh(ImageIds *image_ids) { std::string pool_name = m_remote_io_ctx.get_pool_name(); rbd_mirror_mode_t mirror_mode; - int r = librbd::mirror_mode_get(m_remote_io_ctx, &mirror_mode); + int r = librbd::api::Mirror<>::mode_get(m_remote_io_ctx, &mirror_mode); if (r < 0) { derr << "could not tell whether mirroring was enabled for " << pool_name << ": " << cpp_strerror(r) << dendl; @@ -100,7 +101,7 @@ int PoolWatcher::refresh(ImageIds *image_ids) { } std::map images_map; - r = librbd::list_images_v2(m_remote_io_ctx, images_map); + r = librbd::api::Image<>::list_images(m_remote_io_ctx, &images_map); if (r < 0) { derr << "error retrieving image names from pool " << pool_name << ": " << cpp_strerror(r) << dendl; diff --git a/src/tools/rbd_mirror/Replayer.cc b/src/tools/rbd_mirror/Replayer.cc index 01750aaa777..fd798fa887c 100644 --- a/src/tools/rbd_mirror/Replayer.cc +++ b/src/tools/rbd_mirror/Replayer.cc @@ -13,10 +13,12 @@ #include "include/stringify.h" #include "cls/rbd/cls_rbd_client.h" #include "global/global_context.h" +#include "librbd/internal.h" #include "librbd/Utils.h" #include "librbd/Watcher.h" -#include "librbd/internal.h" +#include "librbd/api/Mirror.h" #include "InstanceWatcher.h" +#include "LeaderWatcher.h" #include "Replayer.h" #include "Threads.h" @@ -389,7 +391,7 @@ int Replayer::init_rados(const std::string &cluster_name, void Replayer::init_local_mirroring_images() { rbd_mirror_mode_t mirror_mode; - int r = librbd::mirror_mode_get(m_local_io_ctx, &mirror_mode); + int r = librbd::api::Mirror<>::mode_get(m_local_io_ctx, &mirror_mode); if (r < 0) { derr << "could not tell whether mirroring was enabled for " << m_local_io_ctx.get_pool_name() << ": " << cpp_strerror(r) << dendl;