From: Dongsheng Yang Date: Thu, 17 Nov 2016 09:19:25 +0000 (-0500) Subject: librbd: Asynchronous image clone X-Git-Tag: v12.0.1~15^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=4cc5b9265aa958f5d56bd0833d5010115ef13c63;p=ceph.git librbd: Asynchronous image clone Signed-off-by: Dongsheng Yang --- diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index e2c721efe82b..f8973e8b2ea5 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -32,6 +32,7 @@ set(librbd_internal_srcs exclusive_lock/PostAcquireRequest.cc exclusive_lock/PreReleaseRequest.cc exclusive_lock/StandardPolicy.cc + image/CloneRequest.cc image/CloseRequest.cc image/CreateRequest.cc image/OpenRequest.cc diff --git a/src/librbd/image/CloneRequest.cc b/src/librbd/image/CloneRequest.cc new file mode 100644 index 000000000000..6e0ef901a64c --- /dev/null +++ b/src/librbd/image/CloneRequest.cc @@ -0,0 +1,565 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "cls/rbd/cls_rbd_client.h" +#include "cls/rbd/cls_rbd_types.h" +#include "common/dout.h" +#include "common/errno.h" +#include "include/assert.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/image/CloneRequest.h" +#include "librbd/image/CreateRequest.h" +#include "librbd/image/RemoveRequest.h" +#include "librbd/image/RefreshRequest.h" +#include "librbd/mirror/EnableRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::image::CloneRequest: " + +namespace librbd { +namespace image { + +using util::create_rados_callback; +using util::create_context_callback; +using util::create_async_context_callback; + +template +CloneRequest::CloneRequest(I *p_imctx, IoCtx &c_ioctx, const std::string &c_name, + ImageOptions c_options, + const std::string &non_primary_global_image_id, + const std::string &primary_mirror_uuid, + ContextWQ *op_work_queue, Context *on_finish) + : m_p_imctx(p_imctx), m_ioctx(c_ioctx), m_name(c_name), m_opts(c_options), + m_pspec(m_p_imctx->md_ctx.get_id(), m_p_imctx->id, m_p_imctx->snap_id), + m_non_primary_global_image_id(non_primary_global_image_id), + m_primary_mirror_uuid(primary_mirror_uuid), + m_op_work_queue(op_work_queue), m_on_finish(on_finish), + m_use_p_features(true) { + + m_cct = reinterpret_cast(m_ioctx.cct()); + + bool default_format_set; + m_opts.is_set(RBD_IMAGE_OPTION_FORMAT, &default_format_set); + if (!default_format_set) { + m_opts.set(RBD_IMAGE_OPTION_FORMAT, static_cast(2)); + } + + ldout(m_cct, 20) << "clone " << &m_p_imctx->md_ctx << " name " << m_p_imctx->name + << " snap " << m_p_imctx->snap_name << " to child " << &m_ioctx + << " name " << m_name << " opts = " << &m_opts << dendl; + return; +} + +template +void CloneRequest::send() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + validate_options(); +} + +template +void CloneRequest::validate_options() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + uint64_t format = 0; + m_opts.get(RBD_IMAGE_OPTION_FORMAT, &format); + if (format < 2) { + lderr(m_cct) << "format 2 or later required for clone" << dendl; + return complete(-EINVAL); + } + + if (m_opts.get(RBD_IMAGE_OPTION_FEATURES, &m_features) == 0) { + if (m_features & ~RBD_FEATURES_ALL) { + lderr(m_cct) << "librbd does not support requested features" << dendl; + return complete(-ENOSYS); + } + m_use_p_features = false; + } + + send_validate_parent(); +} + +template +void CloneRequest::send_validate_parent() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + if (m_p_imctx->snap_id == CEPH_NOSNAP) { + lderr(m_cct) << "image to be cloned must be a snapshot" << dendl; + return complete(-EINVAL); + } + + if (m_p_imctx->old_format) { + lderr(m_cct) << "parent image must be in new format" << dendl; + return complete(-EINVAL); + } + + int r = 0; + bool snap_protected; + m_p_imctx->snap_lock.get_read(); + m_p_features = m_p_imctx->features; + m_size = m_p_imctx->get_image_size(m_p_imctx->snap_id); + r = m_p_imctx->is_snap_protected(m_p_imctx->snap_id, &snap_protected); + m_p_imctx->snap_lock.put_read(); + + if ((m_p_features & RBD_FEATURE_LAYERING) != RBD_FEATURE_LAYERING) { + lderr(m_cct) << "parent image must support layering" << dendl; + return complete(-ENOSYS); + } + + if (r < 0) { + lderr(m_cct) << "unable to locate parent's snapshot" << dendl; + return complete(r); + } + + if (!snap_protected) { + lderr(m_cct) << "parent snapshot must be protected" << dendl; + return complete(-EINVAL); + } + + if ((m_p_features & RBD_FEATURE_JOURNALING) != 0) { + m_force_non_primary = !m_non_primary_global_image_id.empty(); + using klass = CloneRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_validate_parent>(this); + + Journal::is_tag_owner(m_p_imctx, &m_is_primary, ctx); + return; + } + + send_validate_child(); +} + +template +void CloneRequest::handle_validate_parent(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to determine tag ownership: " << cpp_strerror(r) + << dendl; + return complete(r); + } + + if ((m_p_features & RBD_FEATURE_JOURNALING) != 0) { + if (!m_is_primary && !m_force_non_primary) { + lderr(m_cct) << "parent is non-primary mirrored image" << dendl; + return complete(-EINVAL); + } + } + + send_validate_child(); +} + +template +void CloneRequest::send_validate_child() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + using klass = CloneRequest; + librados::AioCompletion *comp = create_rados_callback(this); + + librados::ObjectReadOperation op; + op.stat(NULL, NULL, NULL); + + int r = m_ioctx.aio_operate(util::old_header_name(m_name), comp, &op, &m_out_bl); + assert(r == 0); + comp->release(); +} + +template +void CloneRequest::handle_validate_child(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r != -ENOENT) { + lderr(m_cct) << "rbd image " << m_name << " already exists" << dendl; + return complete(r); + } + + send_create(); +} + +template +void CloneRequest::send_create() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + if (m_use_p_features) { + m_features = m_p_features; + } + + uint64_t order = m_p_imctx->order; + if (m_opts.get(RBD_IMAGE_OPTION_ORDER, &order) != 0) { + m_opts.set(RBD_IMAGE_OPTION_ORDER, order); + } + if ((m_features & RBD_FEATURE_LAYERING) != RBD_FEATURE_LAYERING) { + lderr(m_cct) << "cloning image must support layering" << dendl; + return complete(-ENOSYS); + } + m_opts.set(RBD_IMAGE_OPTION_FEATURES, m_features); + + using klass = CloneRequest; + Context *ctx = create_context_callback(this); + + RWLock::RLocker snap_locker(m_p_imctx->snap_lock); + m_id = librbd::util::generate_image_id(m_ioctx); + CreateRequest *req = CreateRequest::create( + m_ioctx, m_name, m_id, m_size, m_opts, m_non_primary_global_image_id, + m_primary_mirror_uuid, true, m_op_work_queue, ctx); + req->send(); +} + +template +void CloneRequest::handle_create(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "error creating child: " << cpp_strerror(r) << dendl; + return complete(r); + } + send_open(); +} + +template +void CloneRequest::send_open() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + m_imctx = I::create(m_name, "", NULL, m_ioctx, false); + + using klass = CloneRequest; + Context *ctx = create_context_callback(this); + + m_imctx->state->open(true, ctx); +} + +template +void CloneRequest::handle_open(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "Error opening new image: " << cpp_strerror(r) << dendl; + m_r_saved = r; + return send_remove(); + } + + send_set_parent(); +} + +template +void CloneRequest::send_set_parent() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::set_parent(&op, m_pspec, m_size); + + using klass = CloneRequest; + librados::AioCompletion *comp = + create_rados_callback(this); + int r = m_imctx->md_ctx.aio_operate(m_imctx->header_oid, + comp, &op); + assert(r == 0); + comp->release(); +} + +template +void CloneRequest::handle_set_parent(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "couldn't set parent: " << cpp_strerror(r) << dendl; + m_r_saved = r; + return send_close(); + } + + send_add_child(); +} + +template +void CloneRequest::send_add_child() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + librados::ObjectWriteOperation op; + cls_client::add_child(&op, m_pspec, m_id); + + using klass = CloneRequest; + librados::AioCompletion *comp = + create_rados_callback(this); + int r = m_ioctx.aio_operate(RBD_CHILDREN, comp, &op); + assert(r == 0); + comp->release(); +} + +template +void CloneRequest::handle_add_child(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "couldn't add child: " << cpp_strerror(r) << dendl; + m_r_saved = r; + return send_close(); + } + + send_refresh(); +} + +template +void CloneRequest::send_refresh() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + using klass = CloneRequest; + RefreshRequest *req = RefreshRequest::create( + *m_imctx, false, false, + create_context_callback(this)); + req->send(); +} + +template +void CloneRequest::handle_refresh(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + bool snap_protected; + if (r == 0) { + m_p_imctx->snap_lock.get_read(); + r = m_p_imctx->is_snap_protected(m_p_imctx->snap_id, &snap_protected); + m_p_imctx->snap_lock.put_read(); + } + + if (r < 0 || !snap_protected) { + m_r_saved = -EINVAL; + return send_close(); + } + + send_metadata_list(); +} + +template +void CloneRequest::send_metadata_list() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + librados::ObjectReadOperation op; + cls_client::metadata_list_start(&op, "", 0); + + using klass = CloneRequest; + librados::AioCompletion *comp = + create_rados_callback(this); + m_out_bl.clear(); + m_p_imctx->md_ctx.aio_operate(m_p_imctx->header_oid, + comp, &op, &m_out_bl); + comp->release(); +} + +template +void CloneRequest::handle_metadata_list(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r == 0) { + bufferlist::iterator it = m_out_bl.begin(); + r = cls_client::metadata_list_finish(&it, &m_pairs); + } + + if (r < 0 && r != -EOPNOTSUPP && r != -EIO) { + lderr(m_cct) << "couldn't list metadata: " << cpp_strerror(r) << dendl; + m_r_saved = r; + send_remove_child(); + } else if (r == 0 && !m_pairs.empty()) { + send_metadata_set(); + } else { + get_mirror_mode(); + } +} + +template +void CloneRequest::send_metadata_set() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + librados::ObjectWriteOperation op; + cls_client::metadata_set(&op, m_pairs); + + using klass = CloneRequest; + librados::AioCompletion *comp = + create_rados_callback(this); + int r = m_ioctx.aio_operate(m_imctx->header_oid, comp, &op); + assert(r == 0); + comp->release(); +} + +template +void CloneRequest::handle_metadata_set(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "couldn't set metadata: " << cpp_strerror(r) << dendl; + m_r_saved = r; + send_remove_child(); + } else { + get_mirror_mode(); + } +} + +template +void CloneRequest::get_mirror_mode() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + if (!m_imctx->test_features(RBD_FEATURE_JOURNALING)) { + send_close(); + return; + } + + librados::ObjectReadOperation op; + cls_client::mirror_mode_get_start(&op); + + using klass = CloneRequest; + librados::AioCompletion *comp = + create_rados_callback(this); + m_out_bl.clear(); + m_imctx->md_ctx.aio_operate(RBD_MIRRORING, + comp, &op, &m_out_bl); + comp->release(); +} + +template +void CloneRequest::handle_get_mirror_mode(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r == 0) { + lderr(m_cct) << "failed to retrieve mirror mode: " << cpp_strerror(r) + << dendl; + bufferlist::iterator it = m_out_bl.begin(); + r = cls_client::mirror_mode_get_finish(&it, &m_mirror_mode); + } + + if (r < 0 && r != -ENOENT) { + m_r_saved = r; + send_remove_child(); + } else { + if (m_mirror_mode == cls::rbd::MIRROR_MODE_POOL || + !m_non_primary_global_image_id.empty()) { + send_enable_mirror(); + } else { + send_close(); + } + } +} + +template +void CloneRequest::send_enable_mirror() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + using klass = CloneRequest; + Context *ctx = create_context_callback(this); + + mirror::EnableRequest *req = mirror::EnableRequest::create( + m_imctx->md_ctx, m_id, m_non_primary_global_image_id, + m_imctx->op_work_queue, ctx); + req->send(); +} + +template +void CloneRequest::handle_enable_mirror(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to enable mirroring: " << cpp_strerror(r) + << dendl; + m_r_saved = r; + send_remove_child(); + } else { + send_close(); + } +} + +template +void CloneRequest::send_close() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + assert(m_imctx != nullptr); + + using klass = CloneRequest; + Context *ctx = create_async_context_callback( + *m_imctx, create_context_callback< + klass, &klass::handle_close>(this)); + m_imctx->state->close(ctx); +} + +template +void CloneRequest::handle_close(int r) { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + m_imctx->destroy(); + m_imctx = nullptr; + + if (r < 0) { + lderr(m_cct) << "couldn't close image: " << cpp_strerror(r) << dendl; + return complete(r); + } + + if (m_r_saved == 0) { + complete(0); + } else { + send_remove(); + } +} + +template +void CloneRequest::send_remove_child() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + librados::ObjectWriteOperation op; + cls_client::remove_child(&op, m_pspec, m_id); + + using klass = CloneRequest; + librados::AioCompletion *comp = + create_rados_callback(this); + int r = m_p_imctx->md_ctx.aio_operate(RBD_CHILDREN, comp, &op); + assert(r == 0); + comp->release(); +} + +template +void CloneRequest::handle_remove_child(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "Error removing failed clone from list of children: " + << cpp_strerror(r) << dendl; + } + + send_close(); +} + +template +void CloneRequest::send_remove() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + using klass = CloneRequest; + Context *ctx = create_context_callback(this); + + librbd::image::RemoveRequest<> *req = librbd::image::RemoveRequest<>::create( + m_ioctx, m_name, m_id, false, m_no_op, m_op_work_queue, ctx); + req->send(); +} + +template +void CloneRequest::handle_remove(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "Error removing failed clone: " + << cpp_strerror(r) << dendl; + } + complete(r); +} + +template +void CloneRequest::complete(int r) { + ldout(m_cct, 20) << this << " " << __func__ << " r=" << r << dendl; + + if (r == 0) { + ldout(m_cct, 20) << "done." << dendl; + } + + m_on_finish->complete(r); + delete this; +} + +} //namespace image +} //namespace librbd + +template class librbd::image::CloneRequest; diff --git a/src/librbd/image/CloneRequest.h b/src/librbd/image/CloneRequest.h new file mode 100644 index 000000000000..d4052542306b --- /dev/null +++ b/src/librbd/image/CloneRequest.h @@ -0,0 +1,163 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IMAGE_CLONE_REQUEST_H +#define CEPH_LIBRBD_IMAGE_CLONE_REQUEST_H + +#include "cls/rbd/cls_rbd_types.h" +#include "include/rbd/librbd.hpp" +#include "librbd/internal.h" + +class Context; + +using librados::IoCtx; + +namespace librbd { +namespace image { + +template +class CloneRequest { +public: + static CloneRequest *create(ImageCtxT *p_imctx, IoCtx &c_ioctx, const std::string &c_name, + ImageOptions c_options, + const std::string &non_primary_global_image_id, + const std::string &primary_mirror_uuid, + ContextWQ *op_work_queue, Context *on_finish) { + return new CloneRequest(p_imctx, c_ioctx, c_name, c_options, + non_primary_global_image_id, primary_mirror_uuid, + op_work_queue, on_finish); + } + + void send(); +private: + /** + * @verbatim + * + * + * | + * v + * VALIDATE PARENT + * | + * v + * (error: bottom up) VALIDATE CHILD + * _______<_______ | + * | | v + * | | CREATE IMAGE + * | | | + * | | v (parent_md exists) + * | | OPEN IMAGE. . . . . > . . . . + * v | / | . + * | REMOVE IMAGE<--------/ v . + * | | SET PARENT IN HEADER . + * | CLOSE IMAGE / | . + * | ^-------<------/ v . + * | |\ UPDATE DIR_CHILDREN. . < . . . + * | | \ / | + * | | *<-----------/ v + * | | REFRESH + * | | / | + * | CLEAN DIR_CHILDREN <--------/ v (meta is empty) + * | |\ GET META IN PARENT . . . . . . . + * | | \ / | . + * v | *<-----------/ v (journaling disabled) . + * | | SET META IN CHILD . . . . . . . v + * | | / | . + * | -------<-------/ v (no need to enable mirror) . + * | | GET MIRROR MODE . . . . . . . v + * | | / | . + * | -------<-------/ v . + * | | ENABLE MIRROR MODE v + * | | / | . + * | -------<-------/ v . + * | CLOSE IMAGE . . . . .< . . . + * | | + * | v + * |_____________>__________________ + * + * @endverbatim + */ + + CloneRequest(ImageCtxT *p_imctx, IoCtx &c_ioctx, const std::string &c_name, + ImageOptions c_options, + const std::string &non_primary_global_image_id, + const std::string &primary_mirror_uuid, + ContextWQ *op_work_queue, Context *on_finish); + + ImageCtxT *m_p_imctx; + IoCtx &m_ioctx; + std::string m_name; + std::string m_id; + ImageOptions m_opts; + ParentSpec m_pspec; + ImageCtxT *m_imctx; + cls::rbd::MirrorMode m_mirror_mode = cls::rbd::MIRROR_MODE_DISABLED; + const std::string m_non_primary_global_image_id; + const std::string m_primary_mirror_uuid; + NoOpProgressContext m_no_op; + ContextWQ *m_op_work_queue; + Context *m_on_finish; + + CephContext *m_cct; + bool m_use_p_features; + uint64_t m_p_features; + uint64_t m_features; + map m_pairs; + bufferlist m_out_bl; + uint64_t m_size; + int m_r_saved = 0; + bool m_is_primary; + bool m_force_non_primary; + + void validate_options(); + + void send_validate_parent(); + void handle_validate_parent(int r); + + void send_validate_child(); + void handle_validate_child(int r); + + void send_create(); + void handle_create(int r); + + void send_open(); + void handle_open(int r); + + void send_set_parent(); + void handle_set_parent(int r); + + void send_add_child(); + void handle_add_child(int r); + + void send_refresh(); + void handle_refresh(int r); + + void send_metadata_list(); + void handle_metadata_list(int r); + + void send_metadata_set(); + void handle_metadata_set(int r); + + void get_mirror_mode(); + void handle_get_mirror_mode(int r); + + void send_enable_mirror(); + void handle_enable_mirror(int r); + + void send_close(); + void handle_close(int r); + + void send_remove(); + void handle_remove(int r); + + void send_remove_child(); + void handle_remove_child(int r); + + void complete(int r); +}; + +} //namespace image +} //namespace librbd + +extern template class librbd::image::CloneRequest; + +#endif // CEPH_LIBRBD_IMAGE_CLONE_REQUEST_H