From 76e2d867ef4a04f23b506891627d01f055ab6817 Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Tue, 7 Jun 2016 17:17:54 +0530 Subject: [PATCH] rbd-mirror: asynchronously create mirror images Use the newly instroduced asynchronous image creation state machine (CreateRequest) to create mirrored images. Signed-off-by: Venky Shankar --- src/librbd/internal.cc | 192 ------------------ src/librbd/internal.h | 7 - .../image_replayer/CreateImageRequest.cc | 49 ++--- src/tools/rbd_mirror/image_replayer/Utils.h | 14 -- 4 files changed, 20 insertions(+), 242 deletions(-) diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index 3105968f54946..c77dc0c6baefa 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -1112,198 +1112,6 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force, return 0; } - int create_v2(IoCtx& io_ctx, const char *imgname, uint64_t bid, uint64_t size, - int order, uint64_t features, uint64_t stripe_unit, - uint64_t stripe_count, uint8_t journal_order, - uint8_t journal_splay_width, const std::string &journal_pool, - const std::string &non_primary_global_image_id, - const std::string &primary_mirror_uuid) - { - ostringstream bid_ss; - uint32_t extra; - string id, id_obj, header_oid; - int remove_r; - ostringstream oss; - CephContext *cct = (CephContext *)io_ctx.cct(); - - file_layout_t layout; - bool force_non_primary = !non_primary_global_image_id.empty(); - - int r = validate_pool(io_ctx, cct); - if (r < 0) { - return r; - } - - id_obj = util::id_obj_name(imgname); - - r = io_ctx.create(id_obj, true); - if (r < 0) { - lderr(cct) << "error creating rbd id object: " << cpp_strerror(r) - << dendl; - return r; - } - - extra = rand() % 0xFFFFFFFF; - bid_ss << std::hex << bid << std::hex << extra; - id = bid_ss.str(); - - // ensure the image id won't overflow the fixed block name size - const size_t max_id_length = RBD_MAX_BLOCK_NAME_SIZE - - strlen(RBD_DATA_PREFIX) - 1; - if (id.length() > max_id_length) { - id = id.substr(id.length() - max_id_length); - } - - r = cls_client::set_id(&io_ctx, id_obj, id); - if (r < 0) { - lderr(cct) << "error setting image id: " << cpp_strerror(r) << dendl; - goto err_remove_id; - } - - ldout(cct, 2) << "adding rbd image to directory..." << dendl; - r = cls_client::dir_add_image(&io_ctx, RBD_DIRECTORY, imgname, id); - if (r < 0) { - lderr(cct) << "error adding image to directory: " << cpp_strerror(r) - << dendl; - goto err_remove_id; - } - - oss << RBD_DATA_PREFIX << id; - header_oid = util::header_name(id); - r = cls_client::create_image(&io_ctx, header_oid, size, order, - features, oss.str()); - if (r < 0) { - lderr(cct) << "error writing header: " << cpp_strerror(r) << dendl; - goto err_remove_from_dir; - } - - if ((stripe_unit || stripe_count) && - (stripe_count != 1 || stripe_unit != (1ull << order))) { - r = cls_client::set_stripe_unit_count(&io_ctx, header_oid, - stripe_unit, stripe_count); - if (r < 0) { - lderr(cct) << "error setting striping parameters: " - << cpp_strerror(r) << dendl; - goto err_remove_header; - } - } - - if ((features & RBD_FEATURE_FAST_DIFF) != 0 && - (features & RBD_FEATURE_OBJECT_MAP) == 0) { - lderr(cct) << "cannot use fast diff without object map" << dendl; - goto err_remove_header; - } else if ((features & RBD_FEATURE_OBJECT_MAP) != 0) { - if ((features & RBD_FEATURE_EXCLUSIVE_LOCK) == 0) { - lderr(cct) << "cannot use object map without exclusive lock" << dendl; - goto err_remove_header; - } - - layout = file_layout_t(); - layout.object_size = 1ull << order; - if (stripe_unit == 0 || stripe_count == 0) { - layout.stripe_unit = layout.object_size; - layout.stripe_count = 1; - } else { - layout.stripe_unit = stripe_unit; - layout.stripe_count = stripe_count; - } - - if (!ObjectMap::is_compatible(layout, size)) { - lderr(cct) << "image size not compatible with object map" << dendl; - goto err_remove_header; - } - - librados::ObjectWriteOperation op; - cls_client::object_map_resize(&op, Striper::get_num_objects(layout, size), - OBJECT_NONEXISTENT); - r = io_ctx.operate(ObjectMap::object_map_name(id, CEPH_NOSNAP), &op); - if (r < 0) { - lderr(cct) << "error creating initial object map: " - << cpp_strerror(r) << dendl; - goto err_remove_header; - } - } - - if ((features & RBD_FEATURE_JOURNALING) != 0) { - if ((features & RBD_FEATURE_EXCLUSIVE_LOCK) == 0) { - lderr(cct) << "cannot use journaling without exclusive lock" << dendl; - goto err_remove_object_map; - } - - rbd_mirror_mode_t mirror_mode; - r = librbd::mirror_mode_get(io_ctx, &mirror_mode); - if (r < 0) { - lderr(cct) << "error in retrieving pool mirroring status: " - << cpp_strerror(r) << dendl; - goto err_remove_object_map; - } - - r = Journal<>::create(io_ctx, id, journal_order, journal_splay_width, - journal_pool, force_non_primary, - primary_mirror_uuid); - if (r < 0) { - lderr(cct) << "error creating journal: " << cpp_strerror(r) << dendl; - goto err_remove_object_map; - } - - if (mirror_mode == RBD_MIRROR_MODE_POOL || force_non_primary) { - r = mirror_image_enable(cct, io_ctx, id, non_primary_global_image_id); - if (r < 0) { - lderr(cct) << "error enabling mirroring: " << cpp_strerror(r) - << dendl; - goto err_remove_journal; - } - } - } else if (force_non_primary) { - // journaling should have been enabled - assert(false); - } - - ldout(cct, 2) << "done." << dendl; - return 0; - - err_remove_journal: - if ((features & RBD_FEATURE_JOURNALING) != 0) { - remove_r = Journal<>::remove(io_ctx, id); - if (remove_r < 0) { - lderr(cct) << "error cleaning up journal after creation failed: " - << cpp_strerror(remove_r) << dendl; - } - } - - err_remove_object_map: - if ((features & RBD_FEATURE_OBJECT_MAP) != 0) { - remove_r = ObjectMap::remove(io_ctx, id); - if (remove_r < 0) { - lderr(cct) << "error cleaning up object map after creation failed: " - << cpp_strerror(remove_r) << dendl; - } - } - - err_remove_header: - remove_r = io_ctx.remove(header_oid); - if (remove_r < 0) { - lderr(cct) << "error cleaning up image header after creation failed: " - << cpp_strerror(remove_r) << dendl; - } - err_remove_from_dir: - remove_r = cls_client::dir_remove_image(&io_ctx, RBD_DIRECTORY, - imgname, id); - if (remove_r < 0) { - lderr(cct) << "error cleaning up image from rbd_directory object " - << "after creation failed: " << cpp_strerror(remove_r) - << dendl; - } - err_remove_id: - remove_r = io_ctx.remove(id_obj); - if (remove_r < 0) { - lderr(cct) << "error cleaning up id object after creation failed: " - << cpp_strerror(remove_r) << dendl; - } - - return r; - } - void create_v2(IoCtx& io_ctx, std::string &imgname, uint64_t size, int order, uint64_t features, uint64_t stripe_unit, uint64_t stripe_count, uint8_t journal_order, diff --git a/src/librbd/internal.h b/src/librbd/internal.h index 06b012b9be124..0db0b0bf78022 100644 --- a/src/librbd/internal.h +++ b/src/librbd/internal.h @@ -114,13 +114,6 @@ namespace librbd { ImageOptions& opts, const std::string &non_primary_global_image_id, const std::string &primary_mirror_uuid); - int create_v2(IoCtx& io_ctx, const char *imgname, uint64_t bid, uint64_t size, - int order, uint64_t features, uint64_t stripe_unit, - uint64_t stripe_count, uint8_t journal_order, - uint8_t journal_splay_width, - const std::string &journal_pool, - const std::string &non_primary_global_image_id, - const std::string &primary_mirror_uuid); int clone(IoCtx& p_ioctx, const char *p_name, const char *p_snap_name, IoCtx& c_ioctx, const char *c_name, uint64_t features, int *c_order, diff --git a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc index 5611ca2261157..448069a9ff161 100644 --- a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc @@ -12,6 +12,7 @@ #include "librbd/ImageState.h" #include "librbd/internal.h" #include "librbd/Utils.h" +#include "librbd/image/CreateRequest.h" #define dout_subsys ceph_subsys_rbd_mirror #undef dout_prefix @@ -59,35 +60,25 @@ template void CreateImageRequest::create_image() { dout(20) << dendl; - // TODO: librbd should provide an AIO image creation method -- this is - // blocking so we execute in our worker thread - Context *ctx = new FunctionContext([this](int r) { - // TODO: rbd-mirror should offer a feature mask capability - RWLock::RLocker snap_locker(m_remote_image_ctx->snap_lock); - int order = m_remote_image_ctx->order; - - CephContext *cct = reinterpret_cast(m_local_io_ctx.cct()); - uint64_t journal_order = cct->_conf->rbd_journal_order; - uint64_t journal_splay_width = cct->_conf->rbd_journal_splay_width; - std::string journal_pool = cct->_conf->rbd_journal_pool; - - // NOTE: bid is 64bit but overflow will result due to - // RBD_MAX_BLOCK_NAME_SIZE being too small - librados::Rados rados(m_local_io_ctx); - uint64_t bid = rados.get_instance_id(); - - r = utils::create_image(m_local_io_ctx, m_remote_image_ctx, - m_local_image_name.c_str(), bid, - m_remote_image_ctx->size, order, - m_remote_image_ctx->features, - m_remote_image_ctx->stripe_unit, - m_remote_image_ctx->stripe_count, - journal_order, journal_splay_width, - journal_pool, m_global_image_id, - m_remote_mirror_uuid); - handle_create_image(r); - }); - m_work_queue->queue(ctx, 0); + using klass = CreateImageRequest; + Context *ctx = create_context_callback(this); + + RWLock::RLocker snap_locker(m_remote_image_ctx->snap_lock); + int order = m_remote_image_ctx->order; + + CephContext *cct = reinterpret_cast(m_local_io_ctx.cct()); + uint64_t journal_order = cct->_conf->rbd_journal_order; + uint64_t journal_splay_width = cct->_conf->rbd_journal_splay_width; + std::string journal_pool = cct->_conf->rbd_journal_pool; + std::string id = librbd::util::generate_image_id(m_local_io_ctx); + + librbd::image::CreateRequest *req = librbd::image::CreateRequest::create( + m_local_io_ctx, m_local_image_name, id, m_remote_image_ctx->size, + order, m_remote_image_ctx->features, m_remote_image_ctx->stripe_unit, + m_remote_image_ctx->stripe_count, journal_order, journal_splay_width, + journal_pool, m_global_image_id, m_remote_mirror_uuid, + m_remote_image_ctx->op_work_queue, ctx); + req->send(); } template diff --git a/src/tools/rbd_mirror/image_replayer/Utils.h b/src/tools/rbd_mirror/image_replayer/Utils.h index 2fea40ee33476..bf31ad7ee3a20 100644 --- a/src/tools/rbd_mirror/image_replayer/Utils.h +++ b/src/tools/rbd_mirror/image_replayer/Utils.h @@ -17,20 +17,6 @@ namespace utils { // TODO: free-functions used for mocking until create/clone // converted to async state machines -template -int create_image(librados::IoCtx& io_ctx, I *_image_ctx, const char *imgname, - uint64_t bid, uint64_t size, int order, uint64_t features, - uint64_t stripe_unit, uint64_t stripe_count, - uint8_t journal_order, uint8_t journal_splay_width, - const std::string &journal_pool, - const std::string &non_primary_global_image_id, - const std::string &primary_mirror_uuid) { - return librbd::create_v2(io_ctx, imgname, bid, size, order, features, - stripe_unit, stripe_count, journal_order, - journal_splay_width, journal_pool, - non_primary_global_image_id, primary_mirror_uuid); -} - template int clone_image(I *p_imctx, librados::IoCtx& c_ioctx, const char *c_name, librbd::ImageOptions& c_opts, -- 2.39.5