From: Jason Dillaman Date: Fri, 3 Feb 2017 16:16:31 +0000 (-0500) Subject: librbd: additional asynchronous mirror state machines X-Git-Tag: v12.0.1~112^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=6043a9d7b2289bd6d1440832da0f1b15e3e5390d;p=ceph.git librbd: additional asynchronous mirror state machines Signed-off-by: Jason Dillaman --- diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index dc5dda37014a..e2c721efe82b 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -61,8 +61,12 @@ set(librbd_internal_srcs managed_lock/ReacquireRequest.cc managed_lock/ReleaseRequest.cc managed_lock/Utils.cc + mirror/DemoteRequest.cc mirror/DisableRequest.cc mirror/EnableRequest.cc + mirror/GetInfoRequest.cc + mirror/GetStatusRequest.cc + mirror/PromoteRequest.cc object_map/CreateRequest.cc object_map/InvalidateRequest.cc object_map/LockRequest.cc diff --git a/src/librbd/Journal.cc b/src/librbd/Journal.cc index 476f3e00fb2a..050a87b355a5 100644 --- a/src/librbd/Journal.cc +++ b/src/librbd/Journal.cc @@ -97,6 +97,30 @@ struct C_IsTagOwner : public Context { } }; +struct C_GetTagOwner : public Context { + std::string *mirror_uuid; + Context *on_finish; + + Journaler journaler; + cls::journal::Client client; + journal::ImageClientMeta client_meta; + uint64_t tag_tid; + journal::TagData tag_data; + + C_GetTagOwner(librados::IoCtx &io_ctx, const std::string &image_id, + std::string *mirror_uuid, Context *on_finish) + : mirror_uuid(mirror_uuid), on_finish(on_finish), + journaler(io_ctx, image_id, Journal<>::IMAGE_CLIENT_ID, {}) { + } + + virtual void finish(int r) { + if (r >= 0) { + *mirror_uuid = tag_data.mirror_uuid; + } + on_finish->complete(r); + } +}; + template struct GetTagsRequest { CephContext *cct; @@ -433,28 +457,28 @@ int Journal::reset(librados::IoCtx &io_ctx, const std::string &image_id) { template int Journal::is_tag_owner(I *image_ctx, bool *is_tag_owner) { - return Journal<>::is_tag_owner(image_ctx->md_ctx, image_ctx->id, - is_tag_owner); + return Journal::is_tag_owner(image_ctx->md_ctx, image_ctx->id, + is_tag_owner, image_ctx->op_work_queue); } template -int Journal::is_tag_owner(IoCtx& io_ctx, std::string& image_id, - bool *is_tag_owner) { - std::string mirror_uuid; - int r = get_tag_owner(io_ctx, image_id, &mirror_uuid); +int Journal::is_tag_owner(librados::IoCtx& io_ctx, std::string& image_id, + bool *is_tag_owner, ContextWQ *op_work_queue) { + C_SaferCond ctx; + Journal::is_tag_owner(io_ctx, image_id, is_tag_owner, op_work_queue, &ctx); + + int r = ctx.wait(); if (r < 0) { return r; } - - *is_tag_owner = (mirror_uuid == LOCAL_MIRROR_UUID); - return 0; + return r; } template void Journal::is_tag_owner(I *image_ctx, bool *owner, Context *on_finish) { - is_tag_owner(image_ctx->md_ctx, image_ctx->id, owner, - image_ctx->op_work_queue, on_finish); + Journal::is_tag_owner(image_ctx->md_ctx, image_ctx->id, owner, + image_ctx->op_work_queue, on_finish); } template @@ -473,34 +497,29 @@ void Journal::is_tag_owner(librados::IoCtx& io_ctx, std::string& image_id, template int Journal::get_tag_owner(I *image_ctx, std::string *mirror_uuid) { - return get_tag_owner(image_ctx->md_ctx, image_ctx->id, mirror_uuid); -} - -template -int Journal::get_tag_owner(IoCtx& io_ctx, std::string& image_id, - std::string *mirror_uuid) { - CephContext *cct = (CephContext *)io_ctx.cct(); - ldout(cct, 20) << __func__ << dendl; - - Journaler journaler(io_ctx, image_id, IMAGE_CLIENT_ID, {}); - - cls::journal::Client client; - journal::ImageClientMeta client_meta; - uint64_t tag_tid; - journal::TagData tag_data; C_SaferCond get_tags_ctx; - get_tags(cct, &journaler, &client, &client_meta, &tag_tid, - &tag_data, &get_tags_ctx); + get_tag_owner(image_ctx->md_ctx, image_ctx->id, mirror_uuid, + image_ctx->op_work_queue, &get_tags_ctx); int r = get_tags_ctx.wait(); if (r < 0) { return r; } - - *mirror_uuid = tag_data.mirror_uuid; return 0; } +template +void Journal::get_tag_owner(IoCtx& io_ctx, std::string& image_id, + std::string *mirror_uuid, + ContextWQ *op_work_queue, Context *on_finish) { + CephContext *cct = (CephContext *)io_ctx.cct(); + ldout(cct, 20) << __func__ << dendl; + + auto ctx = new C_GetTagOwner(io_ctx, image_id, mirror_uuid, on_finish); + get_tags(cct, &ctx->journaler, &ctx->client, &ctx->client_meta, &ctx->tag_tid, + &ctx->tag_data, create_async_context_callback(op_work_queue, ctx)); +} + template int Journal::request_resync(I *image_ctx) { CephContext *cct = image_ctx->cct; @@ -547,27 +566,22 @@ int Journal::request_resync(I *image_ctx) { } template -int Journal::promote(I *image_ctx) { +void Journal::promote(I *image_ctx, Context *on_finish) { CephContext *cct = image_ctx->cct; ldout(cct, 20) << __func__ << dendl; - C_SaferCond ctx; - auto promote_req = journal::PromoteRequest::create(image_ctx, false, &ctx); + auto promote_req = journal::PromoteRequest::create(image_ctx, false, + on_finish); promote_req->send(); - - return ctx.wait(); } template -int Journal::demote(I *image_ctx) { +void Journal::demote(I *image_ctx, Context *on_finish) { CephContext *cct = image_ctx->cct; ldout(cct, 20) << __func__ << dendl; - C_SaferCond ctx; - auto req = journal::DemoteRequest::create(*image_ctx, &ctx); + auto req = journal::DemoteRequest::create(*image_ctx, on_finish); req->send(); - - return ctx.wait(); } template diff --git a/src/librbd/Journal.h b/src/librbd/Journal.h index 6a60826dc971..4aafefba6d7e 100644 --- a/src/librbd/Journal.h +++ b/src/librbd/Journal.h @@ -104,18 +104,19 @@ public: static int is_tag_owner(ImageCtxT *image_ctx, bool *is_tag_owner); static int is_tag_owner(librados::IoCtx& io_ctx, std::string& image_id, - bool *is_tag_owner); + bool *is_tag_owner, ContextWQ *op_work_queue); static void is_tag_owner(ImageCtxT *image_ctx, bool *is_tag_owner, Context *on_finish); static void is_tag_owner(librados::IoCtx& io_ctx, std::string& image_id, bool *is_tag_owner, ContextWQ *op_work_queue, Context *on_finish); static int get_tag_owner(ImageCtxT *image_ctx, std::string *mirror_uuid); - static int get_tag_owner(librados::IoCtx& io_ctx, std::string& image_id, - std::string *mirror_uuid); + static void get_tag_owner(librados::IoCtx& io_ctx, std::string& image_id, + std::string *mirror_uuid, + ContextWQ *op_work_queue, Context *on_finish); static int request_resync(ImageCtxT *image_ctx); - static int promote(ImageCtxT *image_ctx); - static int demote(ImageCtxT *image_ctx); + static void promote(ImageCtxT *image_ctx, Context *on_finish); + static void demote(ImageCtxT *image_ctx, Context *on_finish); bool is_journal_ready() const; bool is_journal_replaying() const; diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index 81386af70d21..ca2940251283 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -11,8 +11,10 @@ #include "librbd/ImageState.h" #include "librbd/Journal.h" #include "librbd/api/Image.h" +#include "librbd/mirror/DemoteRequest.h" #include "librbd/mirror/DisableRequest.h" #include "librbd/mirror/EnableRequest.h" +#include "librbd/mirror/PromoteRequest.h" #include "librbd/MirroringWatcher.h" #include @@ -263,39 +265,16 @@ int Mirror::image_promote(I *ictx, bool force) { ldout(cct, 20) << "ictx=" << ictx << ", " << "force=" << force << dendl; - int r = ictx->state->refresh_if_required(); - if (r < 0) { - return r; - } + C_SaferCond ctx; + auto req = mirror::PromoteRequest<>::create(*ictx, force, &ctx); + req->send(); - r = validate_mirroring_enabled(ictx); + int r = ctx.wait(); if (r < 0) { + lderr(cct) << "failed to promote image" << dendl; return r; } - std::string mirror_uuid; - r = Journal::get_tag_owner(ictx, &mirror_uuid); - if (r < 0) { - lderr(cct) << "failed to determine tag ownership: " << cpp_strerror(r) - << dendl; - return r; - } else if (mirror_uuid == Journal<>::LOCAL_MIRROR_UUID) { - lderr(cct) << "image is already primary" << dendl; - return -EINVAL; - } else if (mirror_uuid != Journal<>::ORPHAN_MIRROR_UUID && !force) { - lderr(cct) << "image is still primary within a remote cluster" << dendl; - return -EBUSY; - } - - // TODO: need interlock with local rbd-mirror daemon to ensure it has stopped - // replay - - r = Journal::promote(ictx); - if (r < 0) { - lderr(cct) << "failed to promote image: " << cpp_strerror(r) - << dendl; - return r; - } return 0; } @@ -304,82 +283,16 @@ int Mirror::image_demote(I *ictx) { CephContext *cct = ictx->cct; ldout(cct, 20) << "ictx=" << ictx << dendl; - int r = ictx->state->refresh_if_required(); - if (r < 0) { - return r; - } - - r = validate_mirroring_enabled(ictx); - if (r < 0) { - return r; - } - - bool is_primary; - r = Journal::is_tag_owner(ictx, &is_primary); - if (r < 0) { - lderr(cct) << "failed to determine tag ownership: " << cpp_strerror(r) - << dendl; - return r; - } - - if (!is_primary) { - lderr(cct) << "image is not currently the primary" << dendl; - return -EINVAL; - } - - RWLock::RLocker owner_lock(ictx->owner_lock); - if (ictx->exclusive_lock == nullptr) { - lderr(cct) << "exclusive lock is not active" << dendl; - return -EINVAL; - } - - // avoid accepting new requests from peers while we demote - // the image - ictx->exclusive_lock->block_requests(0); - BOOST_SCOPE_EXIT_ALL( (ictx) ) { - if (ictx->exclusive_lock != nullptr) { - ictx->exclusive_lock->unblock_requests(); - } - }; - - C_SaferCond lock_ctx; - ictx->exclusive_lock->acquire_lock(&lock_ctx); - - // don't block holding lock since refresh might be required - ictx->owner_lock.put_read(); - r = lock_ctx.wait(); - ictx->owner_lock.get_read(); + C_SaferCond ctx; + auto req = mirror::DemoteRequest<>::create(*ictx, &ctx); + req->send(); + int r = ctx.wait(); if (r < 0) { - lderr(cct) << "failed to lock image: " << cpp_strerror(r) << dendl; + lderr(cct) << "failed to demote image" << dendl; return r; - } else if (ictx->exclusive_lock == nullptr || - !ictx->exclusive_lock->is_lock_owner()) { - lderr(cct) << "failed to acquire exclusive lock" << dendl; - return -EROFS; - } - - BOOST_SCOPE_EXIT_ALL( (ictx) ) { - C_SaferCond lock_ctx; - ictx->exclusive_lock->release_lock(&lock_ctx); - lock_ctx.wait(); - }; - - RWLock::RLocker snap_locker(ictx->snap_lock); - if (ictx->journal == nullptr) { - lderr(cct) << "journal is not active" << dendl; - return -EINVAL; - } else if (!ictx->journal->is_tag_owner()) { - lderr(cct) << "image is not currently the primary" << dendl; - return -EINVAL; } - r = Journal::demote(ictx); - if (r < 0) { - lderr(cct) << "failed to demote image: " << cpp_strerror(r) - << dendl; - return r; - } return 0; } diff --git a/src/librbd/mirror/DemoteRequest.cc b/src/librbd/mirror/DemoteRequest.cc new file mode 100644 index 000000000000..1fb23a5ddf22 --- /dev/null +++ b/src/librbd/mirror/DemoteRequest.cc @@ -0,0 +1,197 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/DemoteRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/mirror/GetInfoRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::DemoteRequest: " << this \ + << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using librbd::util::create_context_callback; + +template +void DemoteRequest::send() { + get_info(); +} + +template +void DemoteRequest::get_info() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + DemoteRequest, &DemoteRequest::handle_get_info>(this); + auto req = GetInfoRequest::create(m_image_ctx, &m_mirror_image, + &m_promotion_state, ctx); + req->send(); +} + +template +void DemoteRequest::handle_get_info(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } else if (m_mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + lderr(cct) << "mirroring is not currently enabled" << dendl; + finish(-EINVAL); + return; + } else if (m_promotion_state != PROMOTION_STATE_PRIMARY) { + lderr(cct) << "image is not primary" << dendl; + finish(-EINVAL); + return; + } + + acquire_lock(); +} + +template +void DemoteRequest::acquire_lock() { + CephContext *cct = m_image_ctx.cct; + + m_image_ctx.owner_lock.get_read(); + if (m_image_ctx.exclusive_lock == nullptr) { + m_image_ctx.owner_lock.put_read(); + lderr(cct) << "exclusive lock is not active" << dendl; + finish(-EINVAL); + return; + } + + // avoid accepting new requests from peers while we demote + // the image + m_image_ctx.exclusive_lock->block_requests(0); + m_blocked_requests = true; + + if (m_image_ctx.exclusive_lock->is_lock_owner()) { + m_image_ctx.owner_lock.put_read(); + demote(); + return; + } + + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + DemoteRequest, &DemoteRequest::handle_acquire_lock>(this); + m_image_ctx.exclusive_lock->acquire_lock(ctx); + m_image_ctx.owner_lock.put_read(); +} + +template +void DemoteRequest::handle_acquire_lock(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to lock image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_image_ctx.owner_lock.get_read(); + if (m_image_ctx.exclusive_lock == nullptr || + !m_image_ctx.exclusive_lock->is_lock_owner()) { + m_image_ctx.owner_lock.put_read(); + lderr(cct) << "failed to acquire exclusive lock" << dendl; + finish(-EROFS); + return; + } + m_image_ctx.owner_lock.put_read(); + + demote(); +} + +template +void DemoteRequest::demote() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + DemoteRequest, &DemoteRequest::handle_demote>(this); + Journal::demote(&m_image_ctx, ctx); +} + +template +void DemoteRequest::handle_demote(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + m_ret_val = r; + lderr(cct) << "failed to demote image: " << cpp_strerror(r) << dendl; + } + + release_lock(); +} + +template +void DemoteRequest::release_lock() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + m_image_ctx.owner_lock.get_read(); + if (m_image_ctx.exclusive_lock == nullptr) { + m_image_ctx.owner_lock.put_read(); + finish(0); + return; + } + + auto ctx = create_context_callback< + DemoteRequest, &DemoteRequest::handle_release_lock>(this); + m_image_ctx.exclusive_lock->release_lock(ctx); + m_image_ctx.owner_lock.put_read(); +} + +template +void DemoteRequest::handle_release_lock(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to release exclusive lock: " << cpp_strerror(r) + << dendl; + } + + finish(r); +} + +template +void DemoteRequest::finish(int r) { + if (m_ret_val < 0) { + r = m_ret_val; + } + + { + RWLock::RLocker owner_locker(m_image_ctx.owner_lock); + if (m_blocked_requests && m_image_ctx.exclusive_lock != nullptr) { + m_image_ctx.exclusive_lock->unblock_requests(); + } + } + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::DemoteRequest; diff --git a/src/librbd/mirror/DemoteRequest.h b/src/librbd/mirror/DemoteRequest.h new file mode 100644 index 000000000000..a4f6f0c09646 --- /dev/null +++ b/src/librbd/mirror/DemoteRequest.h @@ -0,0 +1,85 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_DEMOTE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_DEMOTE_REQUEST_H + +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/mirror/Types.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { + +template +class DemoteRequest { +public: + static DemoteRequest *create(ImageCtxT &image_ctx, Context *on_finish) { + return new DemoteRequest(image_ctx, on_finish); + } + + DemoteRequest(ImageCtxT &image_ctx, Context *on_finish) + : m_image_ctx(image_ctx), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_INFO + * | + * v + * ACQUIRE_LOCK * * * * + * | * + * v * + * DEMOTE * + * | * + * v * + * RELEASE_LOCK * + * | * + * v * + * < * * * * * + * + * @endverbatim + */ + + ImageCtxT &m_image_ctx; + Context *m_on_finish; + + int m_ret_val = 0; + bool m_blocked_requests = false; + + cls::rbd::MirrorImage m_mirror_image; + PromotionState m_promotion_state; + + void get_info(); + void handle_get_info(int r); + + void acquire_lock(); + void handle_acquire_lock(int r); + + void demote(); + void handle_demote(int r); + + void release_lock(); + void handle_release_lock(int r); + + void finish(int r); + +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::DemoteRequest; + +#endif // CEPH_LIBRBD_MIRROR_DEMOTE_REQUEST_H diff --git a/src/librbd/mirror/GetInfoRequest.cc b/src/librbd/mirror/GetInfoRequest.cc new file mode 100644 index 000000000000..cc2f033495ce --- /dev/null +++ b/src/librbd/mirror/GetInfoRequest.cc @@ -0,0 +1,144 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/GetInfoRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::GetInfoRequest: " << this \ + << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void GetInfoRequest::send() { + refresh_image(); +} + +template +void GetInfoRequest::refresh_image() { + if (!m_image_ctx.state->is_refresh_required()) { + get_mirror_image(); + return; + } + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + GetInfoRequest, &GetInfoRequest::handle_refresh_image>(this); + m_image_ctx.state->refresh(ctx); +} + +template +void GetInfoRequest::handle_refresh_image(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + get_mirror_image(); +} + +template +void GetInfoRequest::get_mirror_image() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_image_get_start(&op, m_image_ctx.id); + + librados::AioCompletion *comp = create_rados_callback< + GetInfoRequest, &GetInfoRequest::handle_get_mirror_image>(this); + int r = m_image_ctx.md_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + assert(r == 0); + comp->release(); +} + +template +void GetInfoRequest::handle_get_mirror_image(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + m_mirror_image->state = cls::rbd::MIRROR_IMAGE_STATE_DISABLED; + *m_promotion_state = PROMOTION_STATE_NON_PRIMARY; + if (r == 0) { + bufferlist::iterator iter = m_out_bl.begin(); + r = cls_client::mirror_image_get_finish(&iter, m_mirror_image); + } + + if (r == -ENOENT || + m_mirror_image->state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + ldout(cct, 20) << "mirroring is disabled" << dendl; + finish(0); + return; + } else if (r < 0) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + get_tag_owner(); +} + +template +void GetInfoRequest::get_tag_owner() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + GetInfoRequest, &GetInfoRequest::handle_get_tag_owner>(this); + Journal::get_tag_owner(m_image_ctx.md_ctx, m_image_ctx.id, + &m_mirror_uuid, m_image_ctx.op_work_queue, ctx); +} + +template +void GetInfoRequest::handle_get_tag_owner(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to determine tag ownership: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_mirror_uuid == Journal<>::LOCAL_MIRROR_UUID) { + *m_promotion_state = PROMOTION_STATE_PRIMARY; + } else if (m_mirror_uuid == Journal<>::ORPHAN_MIRROR_UUID) { + *m_promotion_state = PROMOTION_STATE_ORPHAN; + } + + finish(0); +} + +template +void GetInfoRequest::finish(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::GetInfoRequest; diff --git a/src/librbd/mirror/GetInfoRequest.h b/src/librbd/mirror/GetInfoRequest.h new file mode 100644 index 000000000000..c37ea5818363 --- /dev/null +++ b/src/librbd/mirror/GetInfoRequest.h @@ -0,0 +1,87 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_GET_INFO_REQUEST_H +#define CEPH_LIBRBD_MIRROR_GET_INFO_REQUEST_H + +#include "include/buffer.h" +#include "librbd/mirror/Types.h" +#include + +struct Context; +namespace cls { namespace rbd { struct MirrorImage; } } + +namespace librbd { + +struct ImageCtx; + +namespace mirror { + +template +class GetInfoRequest { +public: + static GetInfoRequest *create(ImageCtxT &image_ctx, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + Context *on_finish) { + return new GetInfoRequest(image_ctx, mirror_image, promotion_state, + on_finish); + } + + GetInfoRequest(ImageCtxT &image_ctx, cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, Context *on_finish) + : m_image_ctx(image_ctx), m_mirror_image(mirror_image), + m_promotion_state(promotion_state), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * REFRESH + * | + * v + * GET_MIRROR_IMAGE + * | + * v + * GET_TAG_OWNER + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT &m_image_ctx; + cls::rbd::MirrorImage *m_mirror_image; + PromotionState *m_promotion_state; + Context *m_on_finish; + + bufferlist m_out_bl; + std::string m_mirror_uuid; + + void refresh_image(); + void handle_refresh_image(int r); + + void get_mirror_image(); + void handle_get_mirror_image(int r); + + void get_tag_owner(); + void handle_get_tag_owner(int r); + + void finish(int r); + +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::GetInfoRequest; + +#endif // CEPH_LIBRBD_MIRROR_GET_INFO_REQUEST_H + diff --git a/src/librbd/mirror/GetStatusRequest.cc b/src/librbd/mirror/GetStatusRequest.cc new file mode 100644 index 000000000000..a798bd86096f --- /dev/null +++ b/src/librbd/mirror/GetStatusRequest.cc @@ -0,0 +1,111 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/GetStatusRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/mirror/GetInfoRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::GetStatusRequest: " << this \ + << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void GetStatusRequest::send() { + *m_mirror_image_status = {cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN, + "status not found"}; + + get_info(); +} + +template +void GetStatusRequest::get_info() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + GetStatusRequest, &GetStatusRequest::handle_get_info>(this); + auto req = GetInfoRequest::create(m_image_ctx, m_mirror_image, + m_promotion_state, ctx); + req->send(); +} + +template +void GetStatusRequest::handle_get_info(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } else if (m_mirror_image->state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + finish(0); + return; + } + + get_status(); +} + +template +void GetStatusRequest::get_status() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_image_status_get_start( + &op, m_mirror_image->global_image_id); + + librados::AioCompletion *comp = create_rados_callback< + GetStatusRequest, &GetStatusRequest::handle_get_status>(this); + int r = m_image_ctx.md_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + assert(r == 0); + comp->release(); +} + +template +void GetStatusRequest::handle_get_status(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r == 0) { + bufferlist::iterator iter = m_out_bl.begin(); + r = cls_client::mirror_image_status_get_finish(&iter, + m_mirror_image_status); + } + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve mirror image status: " << cpp_strerror(r) + << dendl; + finish(r); + } + + finish(0); +} + +template +void GetStatusRequest::finish(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::GetStatusRequest; diff --git a/src/librbd/mirror/GetStatusRequest.h b/src/librbd/mirror/GetStatusRequest.h new file mode 100644 index 000000000000..4c1a81f08f02 --- /dev/null +++ b/src/librbd/mirror/GetStatusRequest.h @@ -0,0 +1,85 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_GET_STATUS_REQUEST_H +#define CEPH_LIBRBD_MIRROR_GET_STATUS_REQUEST_H + +#include "include/buffer.h" +#include "librbd/mirror/Types.h" +#include + +struct Context; +namespace cls { namespace rbd { struct MirrorImage; } } +namespace cls { namespace rbd { struct MirrorImageStatus; } } + +namespace librbd { + +struct ImageCtx; + +namespace mirror { + +template +class GetStatusRequest { +public: + static GetStatusRequest *create(ImageCtxT &image_ctx, + cls::rbd::MirrorImageStatus *status, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + Context *on_finish) { + return new GetStatusRequest(image_ctx, status, mirror_image, + promotion_state, on_finish); + } + + GetStatusRequest(ImageCtxT &image_ctx, cls::rbd::MirrorImageStatus *status, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, Context *on_finish) + : m_image_ctx(image_ctx), m_mirror_image_status(status), + m_mirror_image(mirror_image), m_promotion_state(promotion_state), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_INFO + * | + * v + * GET_STATUS + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT &m_image_ctx; + cls::rbd::MirrorImageStatus *m_mirror_image_status; + cls::rbd::MirrorImage *m_mirror_image; + PromotionState *m_promotion_state; + Context *m_on_finish; + + bufferlist m_out_bl; + + void get_info(); + void handle_get_info(int r); + + void get_status(); + void handle_get_status(int r); + + void finish(int r); + +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::GetStatusRequest; + +#endif // CEPH_LIBRBD_MIRROR_GET_STATUS_REQUEST_H + diff --git a/src/librbd/mirror/PromoteRequest.cc b/src/librbd/mirror/PromoteRequest.cc new file mode 100644 index 000000000000..5603cb13423e --- /dev/null +++ b/src/librbd/mirror/PromoteRequest.cc @@ -0,0 +1,103 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/PromoteRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/mirror/GetInfoRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::PromoteRequest: " << this \ + << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using librbd::util::create_context_callback; + +template +void PromoteRequest::send() { + get_info(); +} + +template +void PromoteRequest::get_info() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + PromoteRequest, &PromoteRequest::handle_get_info>(this); + auto req = GetInfoRequest::create(m_image_ctx, &m_mirror_image, + &m_promotion_state, ctx); + req->send(); +} + +template +void PromoteRequest::handle_get_info(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } else if (m_mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + lderr(cct) << "mirroring is not currently enabled" << dendl; + finish(-EINVAL); + return; + } else if (m_promotion_state == PROMOTION_STATE_PRIMARY) { + lderr(cct) << "image is already primary" << dendl; + finish(-EINVAL); + return; + } else if (m_promotion_state == PROMOTION_STATE_NON_PRIMARY && !m_force) { + lderr(cct) << "image is still primary within a remote cluster" << dendl; + finish(-EBUSY); + return; + } + + promote(); +} + +template +void PromoteRequest::promote() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + PromoteRequest, &PromoteRequest::handle_promote>(this); + Journal::promote(&m_image_ctx, ctx); +} + +template +void PromoteRequest::handle_promote(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to promote image: " << cpp_strerror(r) + << dendl; + } + + finish(r); +} + +template +void PromoteRequest::finish(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::PromoteRequest; diff --git a/src/librbd/mirror/PromoteRequest.h b/src/librbd/mirror/PromoteRequest.h new file mode 100644 index 000000000000..185dc22b0ed5 --- /dev/null +++ b/src/librbd/mirror/PromoteRequest.h @@ -0,0 +1,75 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_PROMOTE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_PROMOTE_REQUEST_H + +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/mirror/Types.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { + +template +class PromoteRequest { +public: + static PromoteRequest *create(ImageCtxT &image_ctx, bool force, + Context *on_finish) { + return new PromoteRequest(image_ctx, force, on_finish); + } + + PromoteRequest(ImageCtxT &image_ctx, bool force, Context *on_finish) + : m_image_ctx(image_ctx), m_force(force), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_INFO + * | + * v + * GET_TAG_OWNER + * | + * v + * PROMOTE + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT &m_image_ctx; + bool m_force; + Context *m_on_finish; + + cls::rbd::MirrorImage m_mirror_image; + PromotionState m_promotion_state; + + void get_info(); + void handle_get_info(int r); + + void promote(); + void handle_promote(int r); + + void finish(int r); + +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::PromoteRequest; + +#endif // CEPH_LIBRBD_MIRROR_PROMOTE_REQUEST_H diff --git a/src/librbd/mirror/Types.h b/src/librbd/mirror/Types.h new file mode 100644 index 000000000000..38511bdb7c15 --- /dev/null +++ b/src/librbd/mirror/Types.h @@ -0,0 +1,20 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_TYPES_H +#define CEPH_LIBRBD_MIRROR_TYPES_H + +namespace librbd { +namespace mirror { + +enum PromotionState { + PROMOTION_STATE_PRIMARY, + PROMOTION_STATE_NON_PRIMARY, + PROMOTION_STATE_ORPHAN +}; + +} // namespace mirror +} // namespace librbd + +#endif // CEPH_LIBRBD_MIRROR_TYPES_H + diff --git a/src/tools/rbd_mirror/ImageDeleter.cc b/src/tools/rbd_mirror/ImageDeleter.cc index fd420872a62b..1c10147d6880 100644 --- a/src/tools/rbd_mirror/ImageDeleter.cc +++ b/src/tools/rbd_mirror/ImageDeleter.cc @@ -281,7 +281,7 @@ bool ImageDeleter::process_image_delete() { bool is_primary = false; r = Journal<>::is_tag_owner(ioctx, m_active_delete->local_image_id, - &is_primary); + &is_primary, m_work_queue); if (r < 0 && r != -ENOENT) { derr << "error retrieving image primary info: " << cpp_strerror(r) << dendl;