From d5fa466d0a73c6edaaf68dc6901e2d9d6a2cc9b2 Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Mon, 4 Jul 2016 10:43:28 +0530 Subject: [PATCH] librbd / journal: async journal creation state machine Signed-off-by: Venky Shankar --- src/librbd/CMakeLists.txt | 1 + src/librbd/Journal.cc | 64 ++------ src/librbd/Journal.h | 5 +- src/librbd/Makefile.am | 2 + src/librbd/internal.cc | 3 +- src/librbd/journal/CreateRequest.cc | 235 +++++++++++++++++++++++++++ src/librbd/journal/CreateRequest.h | 106 ++++++++++++ src/test/librbd/test_mock_Journal.cc | 34 ++++ 8 files changed, 397 insertions(+), 53 deletions(-) create mode 100644 src/librbd/journal/CreateRequest.cc create mode 100644 src/librbd/journal/CreateRequest.h diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index 889ff5488f2de..dea8f9fee6f42 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -37,6 +37,7 @@ set(librbd_internal_srcs image_watcher/Notifier.cc image_watcher/NotifyLockOwner.cc journal/RemoveRequest.cc + journal/CreateRequest.cc journal/Replay.cc journal/StandardPolicy.cc object_map/InvalidateRequest.cc diff --git a/src/librbd/Journal.cc b/src/librbd/Journal.cc index 74a4f96fa89fe..3d249b69b6870 100644 --- a/src/librbd/Journal.cc +++ b/src/librbd/Journal.cc @@ -18,6 +18,7 @@ #include "common/WorkQueue.h" #include "include/rados/librados.hpp" #include "librbd/journal/RemoveRequest.h" +#include "librbd/journal/CreateRequest.h" #include @@ -336,58 +337,25 @@ bool Journal::is_journal_supported(I &image_ctx) { template int Journal::create(librados::IoCtx &io_ctx, const std::string &image_id, - uint8_t order, uint8_t splay_width, - const std::string &object_pool, bool non_primary, - const std::string &primary_mirror_uuid) { + uint8_t order, uint8_t splay_width, + const std::string &object_pool) { CephContext *cct = reinterpret_cast(io_ctx.cct()); ldout(cct, 5) << __func__ << ": image=" << image_id << dendl; - librados::Rados rados(io_ctx); - int64_t pool_id = -1; - if (!object_pool.empty()) { - IoCtx data_io_ctx; - int r = rados.ioctx_create(object_pool.c_str(), data_io_ctx); - if (r != 0) { - lderr(cct) << __func__ << ": " - << "failed to create journal: " - << "error opening journal objects pool '" << object_pool - << "': " << cpp_strerror(r) << dendl; - return r; - } - pool_id = data_io_ctx.get_id(); - } - - Journaler journaler(io_ctx, image_id, IMAGE_CLIENT_ID, {}); - - int r = journaler.create(order, splay_width, pool_id); - if (r < 0) { - lderr(cct) << __func__ << ": " - << "failed to create journal: " << cpp_strerror(r) << dendl; - return r; - } - - cls::journal::Client client; - cls::journal::Tag tag; - journal::TagData tag_data; - - assert(non_primary ^ primary_mirror_uuid.empty()); - std::string mirror_uuid = (non_primary ? primary_mirror_uuid : - LOCAL_MIRROR_UUID); - r = allocate_journaler_tag(cct, &journaler, client, - cls::journal::Tag::TAG_CLASS_NEW, - tag_data, mirror_uuid, &tag); + C_SaferCond cond; + journal::TagData tag_data(LOCAL_MIRROR_UUID); + ContextWQ op_work_queue("librbd::op_work_queue", + cct->_conf->rbd_op_thread_timeout, + ImageCtx::get_thread_pool_instance(cct)); + journal::CreateRequest *req = journal::CreateRequest::create( + io_ctx, image_id, order, splay_width, object_pool, cls::journal::Tag::TAG_CLASS_NEW, + tag_data, IMAGE_CLIENT_ID, &op_work_queue, &cond); + req->send(); - bufferlist client_data; - ::encode(journal::ClientData{journal::ImageClientMeta{tag.tag_class}}, - client_data); + int r = cond.wait(); + op_work_queue.drain(); - r = journaler.register_client(client_data); - if (r < 0) { - lderr(cct) << __func__ << ": " - << "failed to register client: " << cpp_strerror(r) << dendl; - return r; - } - return 0; + return r; } template @@ -453,7 +421,7 @@ int Journal::reset(librados::IoCtx &io_ctx, const std::string &image_id) { return r; } - r = create(io_ctx, image_id, order, splay_width, pool_name, false, ""); + r = create(io_ctx, image_id, order, splay_width, pool_name); if (r < 0) { lderr(cct) << __func__ << ": " << "failed to create journal: " << cpp_strerror(r) << dendl; diff --git a/src/librbd/Journal.h b/src/librbd/Journal.h index ec3b328d88d40..7777e8090a851 100644 --- a/src/librbd/Journal.h +++ b/src/librbd/Journal.h @@ -94,9 +94,8 @@ public: static bool is_journal_supported(ImageCtxT &image_ctx); static int create(librados::IoCtx &io_ctx, const std::string &image_id, - uint8_t order, uint8_t splay_width, - const std::string &object_pool, bool non_primary, - const std::string &primary_mirror_uuid); + uint8_t order, uint8_t splay_width, + const std::string &object_pool); static int remove(librados::IoCtx &io_ctx, const std::string &image_id); static int reset(librados::IoCtx &io_ctx, const std::string &image_id); diff --git a/src/librbd/Makefile.am b/src/librbd/Makefile.am index d961203f43226..d1075cca40496 100644 --- a/src/librbd/Makefile.am +++ b/src/librbd/Makefile.am @@ -42,6 +42,7 @@ librbd_internal_la_SOURCES = \ librbd/image_watcher/Notifier.cc \ librbd/image_watcher/NotifyLockOwner.cc \ librbd/journal/RemoveRequest.cc \ + librbd/journal/CreateRequest.cc \ librbd/journal/Replay.cc \ librbd/journal/StandardPolicy.cc \ librbd/object_map/InvalidateRequest.cc \ @@ -131,6 +132,7 @@ noinst_HEADERS += \ librbd/image_watcher/Notifier.h \ librbd/image_watcher/NotifyLockOwner.h \ librbd/journal/RemoveRequest.h \ + librbd/journal/CreateRequest.h \ librbd/journal/Policy.h \ librbd/journal/Replay.h \ librbd/journal/StandardPolicy.h \ diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index f32734558033f..b033b7b6d2efb 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -1905,8 +1905,7 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force, features_mask |= RBD_FEATURE_EXCLUSIVE_LOCK; r = Journal<>::create(ictx->md_ctx, ictx->id, ictx->journal_order, - ictx->journal_splay_width, - ictx->journal_pool, false, ""); + ictx->journal_splay_width, ictx->journal_pool); if (r < 0) { lderr(cct) << "error creating image journal: " << cpp_strerror(r) << dendl; diff --git a/src/librbd/journal/CreateRequest.cc b/src/librbd/journal/CreateRequest.cc new file mode 100644 index 0000000000000..b897fb503b1ef --- /dev/null +++ b/src/librbd/journal/CreateRequest.cc @@ -0,0 +1,235 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/dout.h" +#include "common/errno.h" +#include "include/assert.h" +#include "librbd/Utils.h" +#include "common/Timer.h" +#include "common/WorkQueue.h" +#include "journal/Settings.h" +#include "librbd/journal/CreateRequest.h" +#include "librbd/journal/RemoveRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::Journal::CreateRequest: " + +namespace librbd { + +using util::create_context_callback; + +namespace journal { + +template +CreateRequest::CreateRequest(IoCtx &ioctx, const std::string &imageid, + uint8_t order, uint8_t splay_width, + const std::string &object_pool, + uint64_t tag_class, TagData &tag_data, + const std::string &client_id, + ContextWQ *op_work_queue, + Context *on_finish) + : m_image_id(imageid), m_order(order), m_splay_width(splay_width), + m_object_pool(object_pool), m_tag_class(tag_class), m_tag_data(tag_data), + m_image_client_id(client_id), m_op_work_queue(op_work_queue), + m_on_finish(on_finish) { + m_ioctx.dup(ioctx); + m_cct = reinterpret_cast(m_ioctx.cct()); +} + +template +void CreateRequest::send() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + if (m_order > 64 || m_order < 12) { + lderr(m_cct) << "order must be in the range [12, 64]" << dendl; + complete(-EDOM); + return; + } + if (m_splay_width == 0) { + complete(-EINVAL); + return; + } + + get_pool_id(); +} + +template +void CreateRequest::get_pool_id() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + if (m_object_pool.empty()) { + create_journal(); + return; + } + + librados::Rados rados(m_ioctx); + IoCtx data_ioctx; + int r = rados.ioctx_create(m_object_pool.c_str(), data_ioctx); + if (r != 0) { + lderr(m_cct) << "failed to create journal: " + << "error opening journal object pool '" << m_object_pool + << "': " << cpp_strerror(r) << dendl; + complete(r); + return; + } + + m_pool_id = data_ioctx.get_id(); + create_journal(); +} + +template +void CreateRequest::create_journal() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + ImageCtx::get_timer_instance(m_cct, &m_timer, &m_timer_lock); + m_journaler = new Journaler(m_op_work_queue, m_timer, m_timer_lock, + m_ioctx, m_image_id, m_image_client_id, {}); + + using klass = CreateRequest; + Context *ctx = create_context_callback(this); + + m_journaler->create(m_order, m_splay_width, m_pool_id, ctx); +} + +template +Context *CreateRequest::handle_create_journal(int *result) { + ldout(m_cct, 20) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(m_cct) << "failed to create journal: " << cpp_strerror(*result) << dendl; + shut_down_journaler(*result); + return nullptr; + } + + allocate_journal_tag(); + return nullptr; +} + +template +void CreateRequest::allocate_journal_tag() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + using klass = CreateRequest; + Context *ctx = create_context_callback(this); + + ::encode(m_tag_data, m_bl); + m_journaler->allocate_tag(m_tag_class, m_bl, &m_tag, ctx); +} + +template +Context *CreateRequest::handle_journal_tag(int *result) { + ldout(m_cct, 20) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(m_cct) << "failed to allocate tag: " << cpp_strerror(*result) << dendl; + shut_down_journaler(*result); + return nullptr; + } + + register_client(); + return nullptr; +} + +template +void CreateRequest::register_client() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + m_bl.clear(); + ::encode(ClientData{ImageClientMeta{m_tag.tag_class}}, m_bl); + + using klass = CreateRequest; + Context *ctx = create_context_callback(this); + + m_journaler->register_client(m_bl, ctx); +} + +template +Context *CreateRequest::handle_register_client(int *result) { + ldout(m_cct, 20) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(m_cct) << "failed to register client: " << cpp_strerror(*result) << dendl; + } + + shut_down_journaler(*result); + return nullptr; +} + +template +void CreateRequest::shut_down_journaler(int r) { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + m_r_saved = r; + + using klass = CreateRequest; + Context *ctx = create_context_callback(this); + + m_journaler->shut_down(ctx); +} + +template +Context *CreateRequest::handle_journaler_shutdown(int *result) { + ldout(m_cct, 20) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(m_cct) << "failed to shut down journaler: " << cpp_strerror(*result) << dendl; + } + + delete m_journaler; + + if (!m_r_saved) { + complete(0); + return nullptr; + } + + // there was an error during journal creation, so we rollback + // what ever was done. the easiest way to do this is to invoke + // journal remove state machine, although it's not the most + // cleanest approach when it comes to redundancy, but that's + // ok during the failure path. + remove_journal(); + return nullptr; +} + +template +void CreateRequest::remove_journal() { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + using klass = CreateRequest; + Context *ctx = create_context_callback(this); + + RemoveRequest *req = RemoveRequest::create( + m_ioctx, m_image_id, m_image_client_id, m_op_work_queue, ctx); + req->send(); +} + +template +Context *CreateRequest::handle_remove_journal(int *result) { + ldout(m_cct, 20) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(m_cct) << "error cleaning up journal after creation failed: " + << cpp_strerror(*result) << dendl; + } + + complete(m_r_saved); + return nullptr; +} + +template +void CreateRequest::complete(int r) { + ldout(m_cct, 20) << this << " " << __func__ << dendl; + + if (r == 0) { + ldout(m_cct, 20) << "done." << dendl; + } + + m_on_finish->complete(r); + delete this; +} + +} // namespace journal +} // namespace librbd + +template class librbd::journal::CreateRequest; diff --git a/src/librbd/journal/CreateRequest.h b/src/librbd/journal/CreateRequest.h new file mode 100644 index 0000000000000..b8632ad847c7a --- /dev/null +++ b/src/librbd/journal/CreateRequest.h @@ -0,0 +1,106 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_JOURNAL_CREATE_REQUEST_H +#define CEPH_LIBRBD_JOURNAL_CREATE_REQUEST_H + +#include "include/int_types.h" +#include "include/buffer.h" +#include "include/rados/librados.hpp" +#include "include/rbd/librbd.hpp" +#include "common/Mutex.h" +#include "librbd/ImageCtx.h" +#include "journal/Journaler.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "cls/journal/cls_journal_types.h" + +using librados::IoCtx; +using journal::Journaler; + +class Context; +class ContextWQ; +class SafeTimer; + +namespace journal { + class Journaler; +} + +namespace librbd { + +class ImageCtx; + +namespace journal { + +template +class CreateRequest { +public: + static CreateRequest *create(IoCtx &ioctx, const std::string &imageid, + uint8_t order, uint8_t splay_width, + const std::string &object_pool, + uint64_t tag_class, TagData &tag_data, + const std::string &client_id, + ContextWQ *op_work_queue, Context *on_finish) { + return new CreateRequest(ioctx, imageid, order, splay_width, object_pool, + tag_class, tag_data, client_id, op_work_queue, + on_finish); + } + + void send(); + +private: + typedef typename TypeTraits::Journaler Journaler; + + CreateRequest(IoCtx &ioctx, const std::string &imageid, uint8_t order, + uint8_t splay_width, const std::string &object_pool, + uint64_t tag_class, TagData &tag_data, + const std::string &client_id, ContextWQ *op_work_queue, + Context *on_finish); + + IoCtx m_ioctx; + std::string m_image_id; + uint8_t m_order; + uint8_t m_splay_width; + std::string m_object_pool; + uint64_t m_tag_class; + TagData m_tag_data; + std::string m_image_client_id; + ContextWQ *m_op_work_queue; + Context *m_on_finish; + + CephContext *m_cct; + cls::journal::Tag m_tag; + bufferlist m_bl; + Journaler *m_journaler; + SafeTimer *m_timer; + Mutex *m_timer_lock; + int m_r_saved; + + int64_t m_pool_id = -1; + + void get_pool_id(); + + void create_journal(); + Context *handle_create_journal(int *result); + + void allocate_journal_tag(); + Context *handle_journal_tag(int *result); + + void register_client(); + Context *handle_register_client(int *result); + + void shut_down_journaler(int r); + Context *handle_journaler_shutdown(int *result); + + void remove_journal(); + Context *handle_remove_journal(int *result); + + void complete(int r); +}; + +} // namespace journal +} // namespace librbd + +extern template class librbd::journal::CreateRequest; + +#endif /* CEPH_LIBRBD_JOURNAL_CREATE_REQUEST_H */ diff --git a/src/test/librbd/test_mock_Journal.cc b/src/test/librbd/test_mock_Journal.cc index 1a8f5817519bd..7c69fe8bcfdba 100644 --- a/src/test/librbd/test_mock_Journal.cc +++ b/src/test/librbd/test_mock_Journal.cc @@ -16,6 +16,7 @@ #include "librbd/Utils.h" #include "librbd/journal/Replay.h" #include "librbd/journal/RemoveRequest.h" +#include "librbd/journal/CreateRequest.h" #include "librbd/journal/Types.h" #include "librbd/journal/TypeTraits.h" #include "gmock/gmock.h" @@ -120,6 +121,39 @@ public: MockRemove *MockRemove::s_instance = nullptr; +struct MockCreate { + static MockCreate *s_instance; + static MockCreate &get_instance() { + assert(s_instance != nullptr); + return *s_instance; + } + + MockCreate() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +template<> +class CreateRequest { +public: + static CreateRequest *create(IoCtx &ioctx, const std::string &imageid, + uint8_t order, uint8_t splay_width, + const std::string &object_pool, + uint64_t tag_class, TagData &tag_data, + const std::string &client_id, + ContextWQ *op_work_queue, Context *on_finish) { + return new CreateRequest(); + } + + void send() { + MockCreate::get_instance().send(); + } +}; + +MockCreate *MockCreate::s_instance = nullptr; + } // namespace journal } // namespace librbd -- 2.39.5