From 6e1718c7581a894cf8e7e2f4d5d4b7001c2ce89b Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Thu, 12 Nov 2015 16:38:35 -0500 Subject: [PATCH] librbd: separate image refresh into async state machines Signed-off-by: Jason Dillaman --- src/CMakeLists.txt | 6 + src/common/Readahead.cc | 28 +- src/common/Readahead.h | 6 +- src/librbd/ExclusiveLock.h | 4 + src/librbd/ImageCtx.cc | 105 +++- src/librbd/ImageCtx.h | 16 +- src/librbd/ImageRefresh.cc | 35 ++ src/librbd/ImageRefresh.h | 33 + src/librbd/Makefile.am | 12 + src/librbd/image/CloseRequest.cc | 258 ++++++++ src/librbd/image/CloseRequest.h | 116 ++++ src/librbd/image/OpenRequest.cc | 373 +++++++++++ src/librbd/image/OpenRequest.h | 106 ++++ src/librbd/image/RefreshParentRequest.cc | 235 +++++++ src/librbd/image/RefreshParentRequest.h | 99 +++ src/librbd/image/RefreshRequest.cc | 762 +++++++++++++++++++++++ src/librbd/image/RefreshRequest.h | 189 ++++++ src/librbd/image/SetSnapRequest.cc | 342 ++++++++++ src/librbd/image/SetSnapRequest.h | 121 ++++ src/librbd/internal.cc | 9 +- 20 files changed, 2839 insertions(+), 16 deletions(-) create mode 100644 src/librbd/ImageRefresh.cc create mode 100644 src/librbd/ImageRefresh.h create mode 100644 src/librbd/image/CloseRequest.cc create mode 100644 src/librbd/image/CloseRequest.h create mode 100644 src/librbd/image/OpenRequest.cc create mode 100644 src/librbd/image/OpenRequest.h create mode 100644 src/librbd/image/RefreshParentRequest.cc create mode 100644 src/librbd/image/RefreshParentRequest.h create mode 100644 src/librbd/image/RefreshRequest.cc create mode 100644 src/librbd/image/RefreshRequest.h create mode 100644 src/librbd/image/SetSnapRequest.cc create mode 100644 src/librbd/image/SetSnapRequest.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c53b02d0276b1..e5ada3be176e5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -880,6 +880,7 @@ if(${WITH_RBD}) librbd/DiffIterate.cc librbd/ExclusiveLock.cc librbd/ImageCtx.cc + librbd/ImageRefresh.cc librbd/ImageWatcher.cc librbd/internal.cc librbd/Journal.cc @@ -892,6 +893,11 @@ if(${WITH_RBD}) librbd/Utils.cc librbd/exclusive_lock/AcquireRequest.cc librbd/exclusive_lock/ReleaseRequest.cc + librbd/image/CloseRequest.cc + librbd/image/OpenRequest.cc + librbd/image/RefreshParentRequest.cc + librbd/image/RefreshRequest.cc + librbd/image/SetSnapRequest.cc librbd/object_map/InvalidateRequest.cc librbd/object_map/LockRequest.cc librbd/object_map/Request.cc diff --git a/src/common/Readahead.cc b/src/common/Readahead.cc index b1ee2e099c6f3..55f74db8c06b5 100644 --- a/src/common/Readahead.cc +++ b/src/common/Readahead.cc @@ -18,8 +18,7 @@ Readahead::Readahead() m_readahead_trigger_pos(0), m_readahead_size(0), m_pending(0), - m_pending_lock("Readahead::m_pending_lock"), - m_pending_cond() { + m_pending_lock("Readahead::m_pending_lock") { } Readahead::~Readahead() { @@ -135,19 +134,34 @@ void Readahead::dec_pending(int count) { assert(m_pending >= count); m_pending -= count; if (m_pending == 0) { - m_pending_cond.Signal(); + std::list pending_waiting(std::move(m_pending_waiting)); + m_pending_lock.Unlock(); + + for (auto ctx : pending_waiting) { + ctx->complete(0); + } + } else { + m_pending_lock.Unlock(); } - m_pending_lock.Unlock(); } void Readahead::wait_for_pending() { + C_SaferCond ctx; + wait_for_pending(&ctx); + ctx.wait(); +} + +void Readahead::wait_for_pending(Context *ctx) { m_pending_lock.Lock(); - while (m_pending > 0) { - m_pending_cond.Wait(m_pending_lock); + if (m_pending > 0) { + m_pending_lock.Unlock(); + m_pending_waiting.push_back(ctx); + return; } m_pending_lock.Unlock(); -} + ctx->complete(0); +} void Readahead::set_trigger_requests(int trigger_requests) { m_lock.Lock(); m_trigger_requests = trigger_requests; diff --git a/src/common/Readahead.h b/src/common/Readahead.h index ffce39f1e9086..75822c5f90544 100644 --- a/src/common/Readahead.h +++ b/src/common/Readahead.h @@ -6,6 +6,7 @@ #include "Mutex.h" #include "Cond.h" +#include /** This class provides common state and logic for code that needs to perform readahead @@ -70,6 +71,7 @@ public: Waits until the pending count reaches 0. */ void wait_for_pending(); + void wait_for_pending(Context *ctx); /** Sets the number of sequential requests necessary to trigger readahead. @@ -146,8 +148,8 @@ private: /// Lock for m_pending Mutex m_pending_lock; - /// Signalled when m_pending reaches 0 - Cond m_pending_cond; + /// Waiters for pending readahead + std::list m_pending_waiting; }; #endif diff --git a/src/librbd/ExclusiveLock.h b/src/librbd/ExclusiveLock.h index 509d8d41e52cd..e849de0ec1814 100644 --- a/src/librbd/ExclusiveLock.h +++ b/src/librbd/ExclusiveLock.h @@ -21,6 +21,10 @@ class ExclusiveLock { public: static const std::string WATCHER_LOCK_TAG; + static ExclusiveLock *create(ImageCtxT &image_ctx) { + return new ExclusiveLock(image_ctx); + } + ExclusiveLock(ImageCtxT &image_ctx); ~ExclusiveLock(); diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc index 9871060fdc95c..f37c8e87d90dd 100644 --- a/src/librbd/ImageCtx.cc +++ b/src/librbd/ImageCtx.cc @@ -69,6 +69,19 @@ struct C_FlushCache : public Context { } }; +struct C_ShutDownCache : public Context { + ImageCtx *image_ctx; + Context *on_finish; + + C_ShutDownCache(ImageCtx *_image_ctx, Context *_on_finish) + : image_ctx(_image_ctx), on_finish(_on_finish) { + } + virtual void finish(int r) { + image_ctx->object_cacher->stop(); + on_finish->complete(r); + } +}; + struct C_InvalidateCache : public Context { ImageCtx *image_ctx; bool purge_on_error; @@ -151,6 +164,7 @@ struct C_InvalidateCache : public Context { object_cacher(NULL), writeback_handler(NULL), object_set(NULL), readahead(), total_bytes_read(0), copyup_finisher(NULL), + exclusive_lock(nullptr), object_map(*this), object_map_ptr(nullptr), aio_work_queue(NULL), op_work_queue(NULL), refresh_in_progress(false), asok_hook(new LibrbdAdminSocketHook(this)) { @@ -202,7 +216,7 @@ struct C_InvalidateCache : public Context { delete asok_hook; } - int ImageCtx::init() { + int ImageCtx::init_legacy() { int r; if (id.length()) { @@ -308,6 +322,71 @@ struct C_InvalidateCache : public Context { return 0; } + void ImageCtx::init() { + assert(!header_oid.empty()); + assert(old_format || !id.empty()); + if (!old_format) { + init_layout(); + } + apply_metadata_confs(); + + string pname = string("librbd-") + id + string("-") + + data_ctx.get_pool_name() + string("-") + name; + if (!snap_name.empty()) { + pname += "-"; + pname += snap_name; + } + + perf_start(pname); + + if (cache) { + Mutex::Locker l(cache_lock); + ldout(cct, 20) << "enabling caching..." << dendl; + writeback_handler = new LibrbdWriteback(this, cache_lock); + + uint64_t init_max_dirty = cache_max_dirty; + if (cache_writethrough_until_flush) + init_max_dirty = 0; + ldout(cct, 20) << "Initial cache settings:" + << " size=" << cache_size + << " num_objects=" << 10 + << " max_dirty=" << init_max_dirty + << " target_dirty=" << cache_target_dirty + << " max_dirty_age=" + << cache_max_dirty_age << dendl; + + object_cacher = new ObjectCacher(cct, pname, *writeback_handler, cache_lock, + NULL, NULL, + cache_size, + 10, /* reset this in init */ + init_max_dirty, + cache_target_dirty, + cache_max_dirty_age, + cache_block_writes_upfront); + + // size object cache appropriately + uint64_t obj = cache_max_dirty_object; + if (!obj) { + obj = MIN(2000, MAX(10, cache_size / 100 / sizeof(ObjectCacher::Object))); + } + ldout(cct, 10) << " cache bytes " << cache_size + << " -> about " << obj << " objects" << dendl; + object_cacher->set_max_objects(obj); + + object_set = new ObjectCacher::ObjectSet(NULL, data_ctx.get_id(), 0); + object_set->return_enoent = true; + object_cacher->start(); + } + + if (clone_copy_on_read) { + copyup_finisher = new Finisher(cct); + copyup_finisher->start(); + } + + readahead.set_trigger_requests(readahead_trigger_requests); + readahead.set_max_readahead_size(readahead_max_bytes); + } + void ImageCtx::init_layout() { if (stripe_unit == 0 || stripe_count == 0) { @@ -752,6 +831,21 @@ struct C_InvalidateCache : public Context { return r; } + void ImageCtx::shut_down_cache(Context *on_finish) { + if (object_cacher == NULL) { + on_finish->complete(0); + return; + } + + RWLock::RLocker owner_locker(owner_lock); + cache_lock.Lock(); + object_cacher->release_set(object_set); + cache_lock.Unlock(); + + C_ShutDownCache *shut_down = new C_ShutDownCache(this, on_finish); + flush_cache(new C_InvalidateCache(this, true, false, shut_down)); + } + int ImageCtx::invalidate_cache(bool purge_on_error) { flush_async_operations(); if (object_cacher == NULL) { @@ -883,6 +977,15 @@ struct C_InvalidateCache : public Context { on_finish->complete(0); } + void ImageCtx::flush_copyup(Context *on_finish) { + if (copyup_finisher == nullptr) { + on_finish->complete(0); + return; + } + + copyup_finisher->queue(on_finish); + } + void ImageCtx::clear_pending_completions() { Mutex::Locker l(completed_reqs_lock); ldout(cct, 10) << "clear pending AioCompletion: count=" diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h index 1641de54b1787..dad549148147d 100644 --- a/src/librbd/ImageCtx.h +++ b/src/librbd/ImageCtx.h @@ -41,13 +41,14 @@ class PerfCounters; namespace librbd { struct ImageCtx; + class AioCompletion; class AioImageRequestWQ; class AsyncOperation; class CopyupRequest; - class LibrbdAdminSocketHook; + template class ExclusiveLock; class ImageWatcher; class Journal; - class AioCompletion; + class LibrbdAdminSocketHook; namespace operation { template class ResizeRequest; @@ -137,6 +138,8 @@ namespace librbd { xlist*> async_requests; std::list async_requests_waiters; + ExclusiveLock *exclusive_lock; + ObjectMap object_map; // TODO ObjectMap *object_map_ptr; @@ -198,7 +201,8 @@ namespace librbd { ImageCtx(const std::string &image_name, const std::string &image_id, const char *snap, IoCtx& p, bool read_only); ~ImageCtx(); - int init(); + int init_legacy(); // TODO + void init(); void init_layout(); void perf_start(std::string name); void perf_stop(); @@ -250,7 +254,8 @@ namespace librbd { void user_flushed(); int flush_cache(); void flush_cache(Context *onfinish); - int shutdown_cache(); + int shutdown_cache(); // TODO + void shut_down_cache(Context *on_finish); int invalidate_cache(bool purge_on_error=false); void invalidate_cache(Context *on_finish); void clear_nonexistence_cache(); @@ -267,6 +272,9 @@ namespace librbd { void cancel_async_requests(); void cancel_async_requests(Context *on_finish); + + void flush_copyup(Context *on_finish); + void apply_metadata_confs(); ObjectMap *create_object_map(); diff --git a/src/librbd/ImageRefresh.cc b/src/librbd/ImageRefresh.cc new file mode 100644 index 0000000000000..224b1277c7c84 --- /dev/null +++ b/src/librbd/ImageRefresh.cc @@ -0,0 +1,35 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/ImageRefresh.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/image/RefreshRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::ImageRefresh: " + +namespace librbd { + +template +ImageRefresh::ImageRefresh(I &image_ctx) : m_image_ctx(image_ctx) { +} + +template +bool ImageRefresh::is_refresh_required() const { + // TODO future entry point for AIO ops -- to replace ictx_check call + return false; +} + +template +void ImageRefresh::refresh(Context *on_finish) { + // TODO simple state machine to restrict to a single in-progress refresh / snap set + image::RefreshRequest *req = image::RefreshRequest::create( + m_image_ctx, on_finish); + req->send(); +} + +} // namespace librbd + +template class librbd::ImageRefresh; diff --git a/src/librbd/ImageRefresh.h b/src/librbd/ImageRefresh.h new file mode 100644 index 0000000000000..4612f96658c10 --- /dev/null +++ b/src/librbd/ImageRefresh.h @@ -0,0 +1,33 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IMAGE_REFRESH_H +#define CEPH_LIBRBD_IMAGE_REFRESH_H + +#include "include/int_types.h" + +class Context; + +namespace librbd { + +class ImageCtx; + +template +class ImageRefresh { +public: + ImageRefresh(ImageCtxT &image_ctx); + + bool is_refresh_required() const; + + void refresh(Context *on_finish); + +private: + ImageCtxT &m_image_ctx; + +}; + +} // namespace librbd + +extern template class librbd::ImageRefresh; + +#endif // CEPH_LIBRBD_IMAGE_REFRESH_H diff --git a/src/librbd/Makefile.am b/src/librbd/Makefile.am index d88973c54f869..f5be6da63ca1f 100644 --- a/src/librbd/Makefile.am +++ b/src/librbd/Makefile.am @@ -19,6 +19,7 @@ librbd_internal_la_SOURCES = \ librbd/DiffIterate.cc \ librbd/ExclusiveLock.cc \ librbd/ImageCtx.cc \ + librbd/ImageRefresh.cc \ librbd/ImageWatcher.cc \ librbd/internal.cc \ librbd/Journal.cc \ @@ -29,6 +30,11 @@ librbd_internal_la_SOURCES = \ librbd/Utils.cc \ librbd/exclusive_lock/AcquireRequest.cc \ librbd/exclusive_lock/ReleaseRequest.cc \ + librbd/image/CloseRequest.cc \ + librbd/image/OpenRequest.cc \ + librbd/image/RefreshParentRequest.cc \ + librbd/image/RefreshRequest.cc \ + librbd/image/SetSnapRequest.cc \ librbd/object_map/InvalidateRequest.cc \ librbd/object_map/LockRequest.cc \ librbd/object_map/Request.cc \ @@ -87,6 +93,7 @@ noinst_HEADERS += \ librbd/DiffIterate.h \ librbd/ExclusiveLock.h \ librbd/ImageCtx.h \ + librbd/ImageRefresh.h \ librbd/ImageWatcher.h \ librbd/internal.h \ librbd/Journal.h \ @@ -102,6 +109,11 @@ noinst_HEADERS += \ librbd/WatchNotifyTypes.h \ librbd/exclusive_lock/AcquireRequest.h \ librbd/exclusive_lock/ReleaseRequest.h \ + librbd/image/CloseRequest.h \ + librbd/image/OpenRequest.h \ + librbd/image/RefreshParentRequest.h \ + librbd/image/RefreshRequest.h \ + librbd/image/SetSnapRequest.h \ librbd/object_map/InvalidateRequest.h \ librbd/object_map/LockRequest.h \ librbd/object_map/Request.h \ diff --git a/src/librbd/image/CloseRequest.cc b/src/librbd/image/CloseRequest.cc new file mode 100644 index 0000000000000..34bc4b1d6c393 --- /dev/null +++ b/src/librbd/image/CloseRequest.cc @@ -0,0 +1,258 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/image/CloseRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "common/WorkQueue.h" +#include "librbd/AioImageRequestWQ.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/ImageWatcher.h" +#include "librbd/ObjectMap.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::image::CloseRequest: " + +namespace librbd { +namespace image { + + +using util::create_async_context_callback; +using util::create_context_callback; + +template +CloseRequest::CloseRequest(I *image_ctx, Context *on_finish) + : m_image_ctx(image_ctx), m_on_finish(on_finish), m_error_result(0), + m_exclusive_lock(nullptr) { + assert(image_ctx != nullptr); +} + +template +void CloseRequest::send() { + // TODO + send_shut_down_aio_queue(); + //send_unregister_image_watcher(); +} + +template +void CloseRequest::send_unregister_image_watcher() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + // prevent incoming requests from our peers + +} + +template +void CloseRequest::handle_unregister_image_watcher(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; + + save_result(r); + if (r < 0) { + lderr(cct) << "failed to unregister image watcher: " << cpp_strerror(r) + << dendl; + } + + send_shut_down_aio_queue(); +} + +template +void CloseRequest::send_shut_down_aio_queue() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + RWLock::RLocker owner_locker(m_image_ctx->owner_lock); + m_image_ctx->aio_work_queue->shut_down(create_context_callback< + CloseRequest, &CloseRequest::handle_shut_down_aio_queue>(this)); +} + +template +void CloseRequest::handle_shut_down_aio_queue(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; + + send_shut_down_exclusive_lock(); +} + +template +void CloseRequest::send_shut_down_exclusive_lock() { + { + RWLock::WLocker owner_locker(m_image_ctx->owner_lock); + RWLock::WLocker snap_locker(m_image_ctx->snap_lock); + std::swap(m_exclusive_lock, m_image_ctx->exclusive_lock); + + if (m_exclusive_lock == nullptr) { + delete m_image_ctx->object_map; + m_image_ctx->object_map = nullptr; + } + } + + if (m_exclusive_lock == nullptr) { + send_flush(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + m_exclusive_lock->shut_down(create_context_callback< + CloseRequest, &CloseRequest::handle_shut_down_exclusive_lock>(this)); +} + +template +void CloseRequest::handle_shut_down_exclusive_lock(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; + + // object map and journal closed during exclusive lock shutdown + assert(m_image_ctx->journal == nullptr); + assert(m_image_ctx->object_map == nullptr); + delete m_exclusive_lock; + + save_result(r); + if (r < 0) { + lderr(cct) << "failed to shut down exclusive lock: " << cpp_strerror(r) + << dendl; + } + send_flush_readahead(); +} + +template +void CloseRequest::send_flush() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + RWLock::RLocker owner_locker(m_image_ctx->owner_lock); + m_image_ctx->flush(create_async_context_callback( + *m_image_ctx, create_context_callback< + CloseRequest, &CloseRequest::handle_flush>(this))); +} + +template +void CloseRequest::handle_flush(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to flush IO: " << cpp_strerror(r) << dendl; + } + send_flush_readahead(); +} + +template +void CloseRequest::send_flush_readahead() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + m_image_ctx->readahead.wait_for_pending(create_context_callback< + CloseRequest, &CloseRequest::handle_flush_readahead>(this)); +} + +template +void CloseRequest::handle_flush_readahead(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; + + send_shut_down_cache(); +} + +template +void CloseRequest::send_shut_down_cache() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + m_image_ctx->shut_down_cache(create_context_callback< + CloseRequest, &CloseRequest::handle_shut_down_cache>(this)); +} + +template +void CloseRequest::handle_shut_down_cache(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; + + save_result(r); + if (r < 0) { + lderr(cct) << "failed to shut down cache: " << cpp_strerror(r) << dendl; + } + send_flush_copyup(); +} + +template +void CloseRequest::send_flush_copyup() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + m_image_ctx->flush_copyup(create_context_callback< + CloseRequest, &CloseRequest::handle_flush_copyup>(this)); +} + +template +void CloseRequest::handle_flush_copyup(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; + send_flush_op_work_queue(); +} + +template +void CloseRequest::send_flush_op_work_queue() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + m_image_ctx->op_work_queue->queue(create_context_callback< + CloseRequest, &CloseRequest::handle_flush_op_work_queue>(this), 0); +} + +template +void CloseRequest::handle_flush_op_work_queue(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; + send_close_parent(); +} + +template +void CloseRequest::send_close_parent() { + if (m_image_ctx->parent == nullptr) { + finish(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + m_image_ctx->parent->state->close(create_async_context_callback( + *m_image_ctx, create_context_callback< + CloseRequest, &CloseRequest::handle_close_parent>(this))); +} + +template +void CloseRequest::handle_close_parent(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; + + delete m_image_ctx->parent; + save_result(r); + if (r < 0) { + lderr(cct) << "error closing parent image: " << cpp_strerror(r) << dendl; + } + finish(); +} + +template +void CloseRequest::finish() { + if (m_image_ctx->image_watcher) { + m_image_ctx->unregister_watch(); + } + + m_on_finish->complete(m_error_result); + delete this; +} + +} // namespace image +} // namespace librbd + +template class librbd::image::CloseRequest; diff --git a/src/librbd/image/CloseRequest.h b/src/librbd/image/CloseRequest.h new file mode 100644 index 0000000000000..1427181dd9658 --- /dev/null +++ b/src/librbd/image/CloseRequest.h @@ -0,0 +1,116 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IMAGE_CLOSE_REQUEST_H +#define CEPH_LIBRBD_IMAGE_CLOSE_REQUEST_H + +#include "include/int_types.h" +#include "librbd/ImageCtx.h" + +class Context; + +namespace librbd { + +class ImageCtx; + +namespace image { + +template +class CloseRequest { +public: + static CloseRequest *create(ImageCtxT *image_ctx, Context *on_finish) { + return new CloseRequest(image_ctx, on_finish); + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * UNREGISTER_IMAGE_WATCHER + * | + * v + * SHUT_DOWN_AIO_WORK_QUEUE . . . + * | . + * v . + * SHUT_DOWN_EXCLUSIVE_LOCK . (exclusive lock + * | . disabled) + * v v + * FLUSH < . . . . . . . . . . . + * | + * v + * FLUSH_READAHEAD + * | + * v + * SHUTDOWN_CACHE + * | + * v + * FLUSH_COPYUP + * | + * v + * FLUSH_OP_WORK_QUEUE . . . . . + * | . + * v . + * CLOSE_PARENT . (no parent) + * | . + * v . + * < . . . . . . . . . . + * + * @endverbatim + */ + + CloseRequest(ImageCtxT *image_ctx, Context *on_finish); + + ImageCtxT *m_image_ctx; + Context *m_on_finish; + + int m_error_result; + + decltype(m_image_ctx->exclusive_lock) m_exclusive_lock; + + void send_unregister_image_watcher(); + void handle_unregister_image_watcher(int r); + + void send_shut_down_aio_queue(); + void handle_shut_down_aio_queue(int r); + + void send_shut_down_exclusive_lock(); + void handle_shut_down_exclusive_lock(int r); + + void send_flush(); + void handle_flush(int r); + + void send_flush_readahead(); + void handle_flush_readahead(int r); + + void send_shut_down_cache(); + void handle_shut_down_cache(int r); + + void send_flush_copyup(); + void handle_flush_copyup(int r); + + void send_flush_op_work_queue(); + void handle_flush_op_work_queue(int r); + + void send_close_parent(); + void handle_close_parent(int r); + + void finish(); + + void save_result(int result) { + if (m_error_result == 0 && result < 0) { + m_error_result = result; + } + } +}; + +} // namespace image +} // namespace librbd + +extern template class librbd::image::CloseRequest; + +#endif // CEPH_LIBRBD_IMAGE_CLOSE_REQUEST_H diff --git a/src/librbd/image/OpenRequest.cc b/src/librbd/image/OpenRequest.cc new file mode 100644 index 0000000000000..f8fb8e977b462 --- /dev/null +++ b/src/librbd/image/OpenRequest.cc @@ -0,0 +1,373 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/image/OpenRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "common/WorkQueue.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/image/CloseRequest.h" +#include "librbd/image/RefreshRequest.h" +#include "librbd/image/SetSnapRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::image::OpenRequest: " + +namespace librbd { +namespace image { + +namespace { + +template +class C_RegisterWatch : public Context { +public: + I &image_ctx; + Context *on_finish; + + C_RegisterWatch(I &image_ctx, Context *on_finish) + : image_ctx(image_ctx), on_finish(on_finish) { + } + + virtual void finish(int r) { + assert(r == 0); + on_finish->complete(image_ctx.register_watch()); + } +}; + +} // anonymous namespace + +using util::create_context_callback; +using util::create_rados_ack_callback; + +template +OpenRequest::OpenRequest(I *image_ctx, Context *on_finish) + : m_image_ctx(image_ctx), m_on_finish(on_finish), m_error_result(0) { +} + +template +void OpenRequest::send() { + send_v2_detect_header(); +} + +template +void OpenRequest::send_v1_detect_header() { + librados::ObjectReadOperation op; + op.stat(NULL, NULL, NULL); + + using klass = OpenRequest; + librados::AioCompletion *comp = + create_rados_ack_callback(this); + m_out_bl.clear(); + m_image_ctx->md_ctx.aio_operate(util::old_header_name(m_image_ctx->name), + comp, &op, &m_out_bl); + comp->release(); +} + +template +Context *OpenRequest::handle_v1_detect_header(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to stat image header: " << cpp_strerror(*result) + << dendl; + send_close_image(*result); + } else { + m_image_ctx->old_format = true; + m_image_ctx->header_oid = util::old_header_name(m_image_ctx->name); + send_register_watch(); + } + return nullptr; +} + +template +void OpenRequest::send_v2_detect_header() { + if (m_image_ctx->id.empty()) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + librados::ObjectReadOperation op; + op.stat(NULL, NULL, NULL); + + using klass = OpenRequest; + librados::AioCompletion *comp = + create_rados_ack_callback(this); + m_out_bl.clear(); + m_image_ctx->md_ctx.aio_operate(util::id_obj_name(m_image_ctx->name), + comp, &op, &m_out_bl); + comp->release(); + } else { + send_v2_get_immutable_metadata(); + } +} + +template +Context *OpenRequest::handle_v2_detect_header(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + if (*result == -ENOENT) { + send_v1_detect_header(); + } else if (*result < 0) { + lderr(cct) << "failed to stat v2 image header: " << cpp_strerror(*result) + << dendl; + send_close_image(*result); + } else { + m_image_ctx->old_format = false; + send_v2_get_id(); + } + return nullptr; +} + +template +void OpenRequest::send_v2_get_id() { + if (m_image_ctx->id.empty()) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + librados::ObjectReadOperation op; + cls_client::get_id_start(&op); + + using klass = OpenRequest; + librados::AioCompletion *comp = + create_rados_ack_callback(this); + m_out_bl.clear(); + m_image_ctx->md_ctx.aio_operate(util::id_obj_name(m_image_ctx->name), + comp, &op, &m_out_bl); + comp->release(); + } else { + send_v2_get_immutable_metadata(); + } +} + +template +Context *OpenRequest::handle_v2_get_id(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + if (*result == 0) { + bufferlist::iterator it = m_out_bl.begin(); + *result = cls_client::get_id_finish(&it, &m_image_ctx->id); + } + if (*result < 0) { + lderr(cct) << "failed to retrieve image id: " << cpp_strerror(*result) + << dendl; + send_close_image(*result); + } else { + send_v2_get_immutable_metadata(); + } + return nullptr; +} + +template +void OpenRequest::send_v2_get_immutable_metadata() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + m_image_ctx->old_format = false; + m_image_ctx->header_oid = util::header_name(m_image_ctx->id); + + librados::ObjectReadOperation op; + cls_client::get_immutable_metadata_start(&op); + + using klass = OpenRequest; + librados::AioCompletion *comp = create_rados_ack_callback< + klass, &klass::handle_v2_get_immutable_metadata>(this); + m_out_bl.clear(); + m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, comp, &op, + &m_out_bl); + comp->release(); +} + +template +Context *OpenRequest::handle_v2_get_immutable_metadata(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + if (*result == 0) { + bufferlist::iterator it = m_out_bl.begin(); + *result = cls_client::get_immutable_metadata_finish( + &it, &m_image_ctx->object_prefix, &m_image_ctx->order); + } + if (*result < 0) { + lderr(cct) << "failed to retreive immutable metadata: " + << cpp_strerror(*result) << dendl; + send_close_image(*result); + } else { + send_v2_get_stripe_unit_count(); + } + + return nullptr; +} + +template +void OpenRequest::send_v2_get_stripe_unit_count() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + librados::ObjectReadOperation op; + cls_client::get_stripe_unit_count_start(&op); + + using klass = OpenRequest; + librados::AioCompletion *comp = create_rados_ack_callback< + klass, &klass::handle_v2_get_stripe_unit_count>(this); + m_out_bl.clear(); + m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, comp, &op, + &m_out_bl); + comp->release(); +} + +template +Context *OpenRequest::handle_v2_get_stripe_unit_count(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + if (*result == 0) { + bufferlist::iterator it = m_out_bl.begin(); + *result = cls_client::get_stripe_unit_count_finish( + &it, &m_image_ctx->stripe_unit, &m_image_ctx->stripe_count); + } + + if (*result == -ENOEXEC || *result == -EINVAL) { + *result = 0; + } + + if (*result < 0) { + lderr(cct) << "failed to read striping metadata: " << cpp_strerror(*result) + << dendl; + send_close_image(*result); + } else { + send_register_watch(); + } + return nullptr; +} + +template +void OpenRequest::send_register_watch() { + m_image_ctx->init(); + + if (!m_image_ctx->read_only) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + // no librados async version of watch + using klass = OpenRequest; + Context *ctx = new C_RegisterWatch( + *m_image_ctx, + create_context_callback(this)); + m_image_ctx->op_work_queue->queue(ctx); + } else { + send_refresh(); + } +} + +template +Context *OpenRequest::handle_register_watch(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to register watch: " << cpp_strerror(*result) + << dendl; + send_close_image(*result); + } else { + send_refresh(); + } + return nullptr; +} + +template +void OpenRequest::send_refresh() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + using klass = OpenRequest; + RefreshRequest *ctx = RefreshRequest::create( + *m_image_ctx, + create_context_callback(this)); + ctx->send(); +} + +template +Context *OpenRequest::handle_refresh(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to refresh image: " << cpp_strerror(*result) + << dendl; + send_close_image(*result); + return nullptr; + } else { + return send_set_snap(result); + } +} + +template +Context *OpenRequest::send_set_snap(int *result) { + if (m_image_ctx->snap_name.empty()) { + *result = 0; + return m_on_finish; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + using klass = OpenRequest; + SetSnapRequest *ctx = SetSnapRequest::create( + *m_image_ctx, m_image_ctx->snap_name, + create_context_callback(this)); + ctx->send(); + return nullptr; +} + +template +Context *OpenRequest::handle_set_snap(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to set image snapshot: " << cpp_strerror(*result) + << dendl; + send_close_image(*result); + return nullptr; + } + + return m_on_finish; +} + +template +void OpenRequest::send_close_image(int error_result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + m_error_result = error_result; + + using klass = OpenRequest; + Context *ctx = create_context_callback( + this); + CloseRequest *req = CloseRequest::create(m_image_ctx, ctx); + req->send(); +} + +template +Context *OpenRequest::handle_close_image(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to close image: " << cpp_strerror(*result) << dendl; + } + if (m_error_result < 0) { + *result = m_error_result; + } + return m_on_finish; +} + +} // namespace image +} // namespace librbd + +template class librbd::image::OpenRequest; diff --git a/src/librbd/image/OpenRequest.h b/src/librbd/image/OpenRequest.h new file mode 100644 index 0000000000000..599fdcefdd227 --- /dev/null +++ b/src/librbd/image/OpenRequest.h @@ -0,0 +1,106 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IMAGE_OPEN_REQUEST_H +#define CEPH_LIBRBD_IMAGE_OPEN_REQUEST_H + +#include "include/int_types.h" +#include "include/buffer.h" + +class Context; + +namespace librbd { + +class ImageCtx; + +namespace image { + +template +class OpenRequest { +public: + static OpenRequest *create(ImageCtxT *image_ctx, Context *on_finish) { + return new OpenRequest(image_ctx, on_finish); + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * | (v1) (read only) + * |-----> V1_DETECT_HEADER . . . . . . . . . . . . . . . . . + * | | . + * | \-------------------------------\ . + * | (v2) | . + * \-----> V2_DETECT_HEADER | . + * | | . + * v | . + * V2_GET_ID | . + * | | . + * v | . + * V2_GET_IMMUTABLE_METADATA | . + * | | . + * v v . + * V2_GET_STRIPE_UNIT_COUNT ----> REGISTER_WATCH . + * . | . + * . (read only) v . + * . . . . . . . . . . . . . > REFRESH < . . . . . + * . | + * . | + * . \--> SET_SNAP + * (no snap) . | + * . v + * . . . > + * ^ + * (on error) | + * * * * * * * > CLOSE --------------------------------/ + * + * @endverbatim + */ + + OpenRequest(ImageCtxT *image_ctx, Context *on_finish); + + ImageCtxT *m_image_ctx; + Context *m_on_finish; + + bufferlist m_out_bl; + int m_error_result; + + void send_v1_detect_header(); + Context *handle_v1_detect_header(int *result); + + void send_v2_detect_header(); + Context *handle_v2_detect_header(int *result); + + void send_v2_get_id(); + Context *handle_v2_get_id(int *result); + + void send_v2_get_immutable_metadata(); + Context *handle_v2_get_immutable_metadata(int *result); + + void send_v2_get_stripe_unit_count(); + Context *handle_v2_get_stripe_unit_count(int *result); + + void send_register_watch(); + Context *handle_register_watch(int *result); + + void send_refresh(); + Context *handle_refresh(int *result); + + Context *send_set_snap(int *result); + Context *handle_set_snap(int *result); + + void send_close_image(int error_result); + Context *handle_close_image(int *result); + +}; + +} // namespace image +} // namespace librbd + +extern template class librbd::image::OpenRequest; + +#endif // CEPH_LIBRBD_IMAGE_OPEN_REQUEST_H diff --git a/src/librbd/image/RefreshParentRequest.cc b/src/librbd/image/RefreshParentRequest.cc new file mode 100644 index 0000000000000..f9ce4f18a891c --- /dev/null +++ b/src/librbd/image/RefreshParentRequest.cc @@ -0,0 +1,235 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/image/RefreshParentRequest.h" +#include "include/rados/librados.hpp" +#include "common/dout.h" +#include "common/errno.h" +#include "common/WorkQueue.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/image/CloseRequest.h" +#include "librbd/image/OpenRequest.h" +#include "librbd/image/SetSnapRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::image::RefreshParentRequest: " + +namespace librbd { +namespace image { + +using util::create_async_context_callback; +using util::create_context_callback; + +template +RefreshParentRequest::RefreshParentRequest(I &child_image_ctx, + const parent_info &parent_md, + Context *on_finish) + : m_child_image_ctx(child_image_ctx), m_parent_md(parent_md), + m_on_finish(on_finish), m_parent_image_ctx(nullptr), + m_parent_snap_id(CEPH_NOSNAP), m_error_result(0) { +} + +template +bool RefreshParentRequest::is_refresh_required(I &child_image_ctx, + const parent_info &parent_md) { + assert(child_image_ctx.snap_lock.is_locked()); + assert(child_image_ctx.parent_lock.is_locked()); + return (is_open_required(child_image_ctx, parent_md) || + is_close_required(child_image_ctx, parent_md)); +} + +template +bool RefreshParentRequest::is_close_required(I &child_image_ctx, + const parent_info &parent_md) { + return (child_image_ctx.parent != nullptr && + (parent_md.spec.pool_id == -1 || parent_md.overlap == 0)); +} + +template +bool RefreshParentRequest::is_open_required(I &child_image_ctx, + const parent_info &parent_md) { + return (parent_md.spec.pool_id > -1 && parent_md.overlap > 0 && + (child_image_ctx.parent == nullptr || + child_image_ctx.parent->md_ctx.get_id() != parent_md.spec.pool_id || + child_image_ctx.parent->id != parent_md.spec.image_id || + child_image_ctx.parent->snap_id != parent_md.spec.snap_id)); +} + +template +void RefreshParentRequest::send() { + if (is_open_required(m_child_image_ctx, m_parent_md)) { + send_open_parent(); + } else { + // parent will be closed (if necessary) during finalize + send_complete(0); + } +} + +template +void RefreshParentRequest::apply() { + if (m_child_image_ctx.parent != nullptr) { + // closing parent image + m_child_image_ctx.clear_nonexistence_cache(); + } + assert(m_child_image_ctx.snap_lock.is_wlocked()); + assert(m_child_image_ctx.parent_lock.is_wlocked()); + std::swap(m_child_image_ctx.parent, m_parent_image_ctx); +} + +template +void RefreshParentRequest::finalize(Context *on_finish) { + CephContext *cct = m_child_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + m_on_finish = on_finish; + if (m_parent_image_ctx != nullptr) { + send_close_parent(); + } else { + send_complete(0); + } +} + +template +void RefreshParentRequest::send_open_parent() { + assert(m_parent_md.spec.pool_id >= 0); + + CephContext *cct = m_child_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + librados::Rados rados(m_child_image_ctx.md_ctx); + + librados::IoCtx parent_io_ctx; + int r = rados.ioctx_create2(m_parent_md.spec.pool_id, parent_io_ctx); + assert(r == 0); + + // since we don't know the image and snapshot name, set their ids and + // reset the snap_name and snap_exists fields after we read the header + m_parent_image_ctx = new I("", m_parent_md.spec.image_id, NULL, parent_io_ctx, + true); + + // set rados flags for reading the parent image + if (m_child_image_ctx.balance_parent_reads) { + m_parent_image_ctx->set_read_flag(librados::OPERATION_BALANCE_READS); + } else if (m_child_image_ctx.localize_parent_reads) { + m_parent_image_ctx->set_read_flag(librados::OPERATION_LOCALIZE_READS); + } + + using klass = RefreshParentRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_open_parent, false>(this); + OpenRequest *req = OpenRequest::create(m_parent_image_ctx, ctx); + req->send(); +} + +template +Context *RefreshParentRequest::handle_open_parent(int *result) { + CephContext *cct = m_child_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << " r=" << *result << dendl; + + save_result(result); + if (*result < 0) { + lderr(cct) << "failed to open parent image: " << cpp_strerror(*result) + << dendl; + send_close_parent(); + return nullptr; + } + + send_set_parent_snap(); + return nullptr; +} + +template +void RefreshParentRequest::send_set_parent_snap() { + assert(m_parent_md.spec.snap_id != CEPH_NOSNAP); + + CephContext *cct = m_child_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + int r; + std::string snap_name; + { + RWLock::RLocker snap_locker(m_parent_image_ctx->snap_lock); + r = m_parent_image_ctx->get_snap_name(m_parent_md.spec.snap_id, &snap_name); + } + + if (r < 0) { + lderr(cct) << "failed to located snapshot: " << cpp_strerror(r) << dendl; + send_complete(r); + return; + } + + using klass = RefreshParentRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_set_parent_snap, false>(this); + SetSnapRequest *req = SetSnapRequest::create( + *m_parent_image_ctx, snap_name, ctx); + req->send(); +} + +template +Context *RefreshParentRequest::handle_set_parent_snap(int *result) { + CephContext *cct = m_child_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << " r=" << *result << dendl; + + save_result(result); + if (*result < 0) { + lderr(cct) << "failed to set parent snapshot: " << cpp_strerror(*result) + << dendl; + send_close_parent(); + return nullptr; + } + + return m_on_finish; +} + +template +void RefreshParentRequest::send_close_parent() { + assert(m_parent_image_ctx != nullptr); + + CephContext *cct = m_child_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + using klass = RefreshParentRequest; + Context *ctx = create_async_context_callback( + m_child_image_ctx, create_context_callback< + klass, &klass::handle_close_parent, false>(this)); + CloseRequest *req = CloseRequest::create(m_parent_image_ctx, ctx); + req->send(); +} + +template +Context *RefreshParentRequest::handle_close_parent(int *result) { + CephContext *cct = m_child_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << " r=" << *result << dendl; + + delete m_parent_image_ctx; + if (*result < 0) { + lderr(cct) << "failed to close parent image: " << cpp_strerror(*result) + << dendl; + } + + if (m_error_result < 0) { + // propagate errors from opening the image + *result = m_error_result; + } else { + // ignore errors from closing the image + *result = 0; + } + + return m_on_finish; +} + +template +void RefreshParentRequest::send_complete(int r) { + CephContext *cct = m_child_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + m_on_finish->complete(r); +} + +} // namespace image +} // namespace librbd + +template class librbd::image::RefreshParentRequest; diff --git a/src/librbd/image/RefreshParentRequest.h b/src/librbd/image/RefreshParentRequest.h new file mode 100644 index 0000000000000..e51d24f48b629 --- /dev/null +++ b/src/librbd/image/RefreshParentRequest.h @@ -0,0 +1,99 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IMAGE_REFRESH_PARENT_REQUEST_H +#define CEPH_LIBRBD_IMAGE_REFRESH_PARENT_REQUEST_H + +#include "include/int_types.h" +#include "librbd/parent_types.h" + +class Context; + +namespace librbd { + +class ImageCtx; + +namespace image { + +template +class RefreshParentRequest { +public: + static RefreshParentRequest *create(ImageCtxT &child_image_ctx, + const parent_info &parent_md, + Context *on_finish) { + return new RefreshParentRequest(child_image_ctx, parent_md, on_finish); + } + + static bool is_refresh_required(ImageCtxT &child_image_ctx, + const parent_info &parent_md); + + void send(); + void apply(); + void finalize(Context *on_finish); + +private: + /** + * @verbatim + * + * + * | + * | (open required) + * |----------------> OPEN_PARENT * * * * * * * * + * | | * + * | v * (on error) + * | SET_PARENT_SNAP * * * * * * + * | | * + * | v * + * \----------------> * + * | * + * | (close required) v + * |-----------------> CLOSE_PARENT + * | | + * | v + * \-----------------> + * + * @endverbatim + */ + + RefreshParentRequest(ImageCtxT &child_image_ctx, const parent_info &parent_md, + Context *on_finish); + + ImageCtxT &m_child_image_ctx; + parent_info m_parent_md; + Context *m_on_finish; + + ImageCtxT *m_parent_image_ctx; + uint64_t m_parent_snap_id; + + int m_error_result; + + static bool is_close_required(ImageCtxT &child_image_ctx, + const parent_info &parent_md); + static bool is_open_required(ImageCtxT &child_image_ctx, + const parent_info &parent_md); + + void send_open_parent(); + Context *handle_open_parent(int *result); + + void send_set_parent_snap(); + Context *handle_set_parent_snap(int *result); + + void send_close_parent(); + Context *handle_close_parent(int *result); + + void send_complete(int r); + + void save_result(int *result) { + if (m_error_result == 0 && *result < 0) { + m_error_result = *result; + } + } + +}; + +} // namespace image +} // namespace librbd + +extern template class librbd::image::RefreshParentRequest; + +#endif // CEPH_LIBRBD_IMAGE_REFRESH_PARENT_REQUEST_H diff --git a/src/librbd/image/RefreshRequest.cc b/src/librbd/image/RefreshRequest.cc new file mode 100644 index 0000000000000..c5edf34059389 --- /dev/null +++ b/src/librbd/image/RefreshRequest.cc @@ -0,0 +1,762 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/image/RefreshRequest.h" +#include "include/stringify.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/lock/cls_lock_client.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/Journal.h" +#include "librbd/ObjectMap.h" +#include "librbd/Utils.h" +#include "librbd/image/RefreshParentRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::image::RefreshRequest: " + +namespace librbd { +namespace image { + +using util::create_rados_ack_callback; +using util::create_context_callback; + +template +RefreshRequest::RefreshRequest(I &image_ctx, Context *on_finish) + : m_image_ctx(image_ctx), m_on_finish(on_finish), m_error_result(0), + m_flush_aio(false), m_exclusive_lock(nullptr), m_object_map(nullptr), + m_journal(nullptr), m_refresh_parent(nullptr) { +} + +template +RefreshRequest::~RefreshRequest() { + delete m_object_map; + + // these require state machine to close + assert(m_exclusive_lock == nullptr); + assert(m_journal == nullptr); + assert(m_refresh_parent == nullptr); +} + +template +void RefreshRequest::send() { + if (m_image_ctx.old_format) { + send_v1_read_header(); + } else { + send_v2_get_mutable_metadata(); + } +} + +template +void RefreshRequest::send_v1_read_header() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + librados::ObjectReadOperation op; + op.read(0, 0, nullptr, nullptr); + + using klass = RefreshRequest; + librados::AioCompletion *comp = create_rados_ack_callback< + klass, &klass::handle_v1_read_header>(this); + m_out_bl.clear(); + int r = m_image_ctx.md_ctx.aio_operate(m_image_ctx.header_oid, comp, &op, + &m_out_bl); + assert(r == 0); + comp->release(); +} + +template +Context *RefreshRequest::handle_v1_read_header(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": " << "r=" << *result << dendl; + + rbd_obj_header_ondisk v1_header; + if (*result < 0) { + return m_on_finish; + } else if (m_out_bl.length() < sizeof(v1_header)) { + lderr(cct) << "v1 header too small" << dendl; + *result = -EIO; + return m_on_finish; + } else if (memcmp(RBD_HEADER_TEXT, m_out_bl.c_str(), + sizeof(RBD_HEADER_TEXT)) != 0) { + lderr(cct) << "unrecognized v1 header" << dendl; + *result = -ENXIO; + return m_on_finish; + } + + memcpy(&v1_header, m_out_bl.c_str(), sizeof(v1_header)); + m_order = v1_header.options.order; + m_size = v1_header.image_size; + m_object_prefix = v1_header.block_name; + send_v1_get_snapshots(); + return nullptr; +} + +template +void RefreshRequest::send_v1_get_snapshots() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + librados::ObjectReadOperation op; + cls_client::old_snapshot_list_start(&op); + + using klass = RefreshRequest; + librados::AioCompletion *comp = create_rados_ack_callback< + klass, &klass::handle_v1_get_snapshots>(this); + m_out_bl.clear(); + int r = m_image_ctx.md_ctx.aio_operate(m_image_ctx.header_oid, comp, &op, + &m_out_bl); + assert(r == 0); + comp->release(); +} + +template +Context *RefreshRequest::handle_v1_get_snapshots(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": " << "r=" << *result << dendl; + + if (*result == 0) { + bufferlist::iterator it = m_out_bl.begin(); + *result = cls_client::old_snapshot_list_finish( + &it, &m_snap_names, &m_snap_sizes, &m_snapc); + } + + if (*result < 0) { + lderr(cct) << "failed to retrieve v1 snapshots: " << cpp_strerror(*result) + << dendl; + return m_on_finish; + } + + if (!m_snapc.is_valid()) { + lderr(cct) << "v1 image snap context is invalid" << dendl; + *result = -EIO; + return m_on_finish; + } + + send_v1_get_locks(); + return nullptr; +} + +template +void RefreshRequest::send_v1_get_locks() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + librados::ObjectReadOperation op; + rados::cls::lock::get_lock_info_start(&op, RBD_LOCK_NAME); + + using klass = RefreshRequest; + librados::AioCompletion *comp = create_rados_ack_callback< + klass, &klass::handle_v1_get_locks>(this); + m_out_bl.clear(); + int r = m_image_ctx.md_ctx.aio_operate(m_image_ctx.header_oid, comp, &op, + &m_out_bl); + assert(r == 0); + comp->release(); +} + +template +Context *RefreshRequest::handle_v1_get_locks(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": " + << "r=" << *result << dendl; + + // If EOPNOTSUPP, treat image as if there are no locks (we can't + // query them). + if (*result == -EOPNOTSUPP) { + *result = 0; + } else if (*result == 0) { + bufferlist::iterator it = m_out_bl.begin(); + ClsLockType lock_type; + *result = rados::cls::lock::get_lock_info_finish(&it, &m_lockers, + &lock_type, &m_lock_tag); + if (*result == 0) { + m_exclusive_locked = (lock_type == LOCK_EXCLUSIVE); + } + } + if (*result < 0) { + lderr(cct) << "failed to retrieve locks: " << cpp_strerror(*result) + << dendl; + return m_on_finish; + } + + apply(); + + return send_flush_aio(); +} + +template +void RefreshRequest::send_v2_get_mutable_metadata() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + uint64_t snap_id; + { + RWLock::RLocker snap_locker(m_image_ctx.snap_lock); + snap_id = m_image_ctx.snap_id; + } + + bool read_only = m_image_ctx.read_only || snap_id != CEPH_NOSNAP; + librados::ObjectReadOperation op; + cls_client::get_mutable_metadata_start(&op, read_only); + + using klass = RefreshRequest; + librados::AioCompletion *comp = create_rados_ack_callback< + klass, &klass::handle_v2_get_mutable_metadata>(this); + m_out_bl.clear(); + int r = m_image_ctx.md_ctx.aio_operate(m_image_ctx.header_oid, comp, &op, + &m_out_bl); + assert(r == 0); + comp->release(); +} + +template +Context *RefreshRequest::handle_v2_get_mutable_metadata(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": " + << "r=" << *result << dendl; + + if (*result == 0) { + bufferlist::iterator it = m_out_bl.begin(); + *result = cls_client::get_mutable_metadata_finish(&it, &m_size, &m_features, + &m_incompatible_features, + &m_lockers, + &m_exclusive_locked, + &m_lock_tag, &m_snapc, + &m_parent_md); + } + if (*result < 0) { + lderr(cct) << "failed to retrieve mutable metadata: " + << cpp_strerror(*result) << dendl; + return m_on_finish; + } + + uint64_t unsupported = m_incompatible_features & ~RBD_FEATURES_ALL; + if (unsupported != 0ULL) { + lderr(cct) << "Image uses unsupported features: " << unsupported << dendl; + *result = -ENOSYS; + return m_on_finish; + } + + send_v2_get_flags(); + return nullptr; +} + +template +void RefreshRequest::send_v2_get_flags() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + librados::ObjectReadOperation op; + cls_client::get_flags_start(&op, m_snapc.snaps); + + using klass = RefreshRequest; + librados::AioCompletion *comp = create_rados_ack_callback< + klass, &klass::handle_v2_get_flags>(this); + m_out_bl.clear(); + int r = m_image_ctx.md_ctx.aio_operate(m_image_ctx.header_oid, comp, &op, + &m_out_bl); + assert(r == 0); + comp->release(); +} + +template +Context *RefreshRequest::handle_v2_get_flags(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": " + << "r=" << *result << dendl; + + if (*result == 0) { + bufferlist::iterator it = m_out_bl.begin(); + cls_client::get_flags_finish(&it, &m_flags, m_snapc.snaps, &m_snap_flags); + } + if (*result == -EOPNOTSUPP) { + // Older OSD doesn't support RBD flags, need to assume the worst + ldout(cct, 10) << "OSD does not support RBD flags, disabling object map " + << "optimizations" << dendl; + m_flags = RBD_FLAG_OBJECT_MAP_INVALID; + if ((m_features & RBD_FEATURE_FAST_DIFF) != 0) { + m_flags |= RBD_FLAG_FAST_DIFF_INVALID; + } + + std::vector default_flags(m_snapc.snaps.size(), m_flags); + m_snap_flags = std::move(default_flags); + } else if (*result == -ENOENT) { + ldout(cct, 10) << "out-of-sync snapshot state detected" << dendl; + send_v2_get_mutable_metadata(); + return nullptr; + } else if (*result < 0) { + lderr(cct) << "failed to retrieve flags: " << cpp_strerror(*result) + << dendl; + return m_on_finish; + } + + return send_v2_get_snapshots(); +} + +template +Context *RefreshRequest::send_v2_get_snapshots() { + if (m_snapc.snaps.empty()) { + m_snap_names.clear(); + m_snap_sizes.clear(); + m_snap_parents.clear(); + m_snap_protection.clear(); + return send_v2_refresh_parent(); + } + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + librados::ObjectReadOperation op; + cls_client::snapshot_list_start(&op, m_snapc.snaps); + + using klass = RefreshRequest; + librados::AioCompletion *comp = create_rados_ack_callback< + klass, &klass::handle_v2_get_snapshots>(this); + m_out_bl.clear(); + int r = m_image_ctx.md_ctx.aio_operate(m_image_ctx.header_oid, comp, &op, + &m_out_bl); + assert(r == 0); + comp->release(); + return nullptr; +} + +template +Context *RefreshRequest::handle_v2_get_snapshots(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": " + << "r=" << *result << dendl; + + if (*result == 0) { + bufferlist::iterator it = m_out_bl.begin(); + *result = cls_client::snapshot_list_finish(&it, m_snapc.snaps, + &m_snap_names, &m_snap_sizes, + &m_snap_parents, + &m_snap_protection); + } + if (*result == -ENOENT) { + ldout(cct, 10) << "out-of-sync snapshot state detected" << dendl; + send_v2_get_mutable_metadata(); + return nullptr; + } else if (*result < 0) { + lderr(cct) << "failed to retrieve snapshots: " << cpp_strerror(*result) + << dendl; + return m_on_finish; + } + + if (!m_snapc.is_valid()) { + lderr(cct) << "image snap context is invalid!" << dendl; + *result = -EIO; + return m_on_finish; + } + + return send_v2_refresh_parent(); +} + +template +Context *RefreshRequest::send_v2_refresh_parent() { + { + RWLock::RLocker snap_locker(m_image_ctx.snap_lock); + RWLock::RLocker parent_locker(m_image_ctx.parent_lock); + + parent_info parent_md; + int r = get_parent_info(m_image_ctx.snap_id, &parent_md); + if (r < 0 || + RefreshParentRequest::is_refresh_required(m_image_ctx, parent_md)) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + using klass = RefreshRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_v2_refresh_parent>(this); + m_refresh_parent = RefreshParentRequest::create( + m_image_ctx, parent_md, ctx); + } + } + + if (m_refresh_parent != nullptr) { + m_refresh_parent->send(); + return nullptr; + } else { + return send_v2_init_exclusive_lock(); + } +} + +template +Context *RefreshRequest::handle_v2_refresh_parent(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to refresh parent image: " << cpp_strerror(*result) + << dendl; + save_result(result); + return send_v2_finalize_refresh_parent(); + } + + return send_v2_init_exclusive_lock(); +} + +template +Context *RefreshRequest::send_v2_init_exclusive_lock() { + if ((m_features & RBD_FEATURE_EXCLUSIVE_LOCK) == 0 || + !m_image_ctx.snap_name.empty() || + m_image_ctx.exclusive_lock != nullptr) { + return send_v2_open_journal(); + } + + // implies exclusive lock dynamically enabled or image open in-progress + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + // TODO need safe shut down + m_exclusive_lock = ExclusiveLock::create(m_image_ctx); + + using klass = RefreshRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_v2_init_exclusive_lock>(this); + + RWLock::RLocker owner_locker(m_image_ctx.owner_lock); + m_exclusive_lock->init(ctx); + return nullptr; +} + +template +Context *RefreshRequest::handle_v2_init_exclusive_lock(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to initialize exclusive lock: " + << cpp_strerror(*result) << dendl; + save_result(result); + return send_v2_finalize_refresh_parent(); + } + + return send_v2_open_journal(); +} + +template +Context *RefreshRequest::send_v2_open_journal() { + if ((m_features & RBD_FEATURE_JOURNALING) == 0 || + m_image_ctx.read_only || + m_image_ctx.journal != nullptr || + m_image_ctx.exclusive_lock == nullptr || + !m_image_ctx.exclusive_lock->is_lock_owner()) { + return send_v2_open_object_map(); + } + + // implies journal dynamically enabled since ExclusiveLock will init + // the journal upon acquiring the lock + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + using klass = RefreshRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_v2_open_journal>(this); + + // TODO need safe close + m_journal = new Journal(m_image_ctx); + m_journal->open(ctx); + return nullptr; +} + +template +Context *RefreshRequest::handle_v2_open_journal(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to initialize journal: " << cpp_strerror(*result) + << dendl; + save_result(result); + return send_v2_finalize_refresh_parent(); + } + + return send_v2_shut_down_exclusive_lock(); +} + +template +Context *RefreshRequest::send_v2_open_object_map() { + if ((m_features & RBD_FEATURE_OBJECT_MAP) == 0 || + m_image_ctx.object_map != nullptr || m_image_ctx.snap_name.empty()) { + return send_v2_finalize_refresh_parent(); + } + + // implies object map dynamically enabled or image open in-progress + // since SetSnapRequest loads the object map for a snapshot and + // ExclusiveLock loads the object map for HEAD + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + for (size_t snap_idx = 0; snap_idx < m_snap_names.size(); ++snap_idx) { + if (m_snap_names[snap_idx] == m_image_ctx.snap_name) { + using klass = RefreshRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_v2_open_object_map>(this); + + m_object_map = m_image_ctx.create_object_map(m_snapc.snaps[snap_idx].val); + m_object_map->open(ctx); + return nullptr; + } + } + + lderr(cct) << "failed to locate snapshot: " << m_image_ctx.snap_name + << dendl; + return send_v2_finalize_refresh_parent(); +} + +template +Context *RefreshRequest::handle_v2_open_object_map(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << *result << dendl; + + assert(*result == 0); + return send_v2_finalize_refresh_parent(); +} + +template +Context *RefreshRequest::send_v2_finalize_refresh_parent() { + apply(); + + if (m_refresh_parent == nullptr) { + return send_v2_shut_down_exclusive_lock(); + } + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + using klass = RefreshRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_v2_finalize_refresh_parent>(this); + m_refresh_parent->finalize(ctx); + return nullptr; +} + +template +Context *RefreshRequest::handle_v2_finalize_refresh_parent(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << *result << dendl; + + assert(m_refresh_parent != nullptr); + delete m_refresh_parent; + m_refresh_parent = nullptr; + + return send_v2_shut_down_exclusive_lock(); +} + +template +Context *RefreshRequest::send_v2_shut_down_exclusive_lock() { + if (m_exclusive_lock == nullptr) { + return send_v2_close_journal(); + } + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + // exclusive lock feature was dynamically disabled + using klass = RefreshRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_v2_shut_down_exclusive_lock>(this); + m_exclusive_lock->shut_down(ctx); + return nullptr; +} + +template +Context *RefreshRequest::handle_v2_shut_down_exclusive_lock(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to shut down exclusive lock: " + << cpp_strerror(*result) << dendl; + save_result(result); + } + + assert(m_exclusive_lock != nullptr); + delete m_exclusive_lock; + m_exclusive_lock = nullptr; + + return send_v2_close_journal(); +} + +template +Context *RefreshRequest::send_v2_close_journal() { + if (m_journal == nullptr) { + return send_flush_aio(); + } + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + // journal feature was dynamically disabled + using klass = RefreshRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_v2_close_journal>(this); + m_journal->close(ctx); + return nullptr; +} + +template +Context *RefreshRequest::handle_v2_close_journal(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + save_result(result); + lderr(cct) << "failed to close journal: " << cpp_strerror(*result) + << dendl; + } + + return send_flush_aio(); +} + +template +Context *RefreshRequest::send_flush_aio() { + if (m_flush_aio) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + RWLock::RLocker owner_lock(m_image_ctx.owner_lock); + using klass = RefreshRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_flush_aio>(this); + m_image_ctx.flush(ctx); + return nullptr; + } + return m_on_finish; +} + +template +Context *RefreshRequest::handle_flush_aio(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to flush pending AIO: " << cpp_strerror(*result) + << dendl; + } + + if (m_error_result < 0) { + *result = m_error_result; + } + return m_on_finish; +} + +template +void RefreshRequest::apply() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << this << " " << __func__ << dendl; + + RWLock::RLocker owner_locker(m_image_ctx.owner_lock); + RWLock::WLocker md_locker(m_image_ctx.md_lock); + + { + Mutex::Locker cache_locker(m_image_ctx.cache_lock); + RWLock::WLocker snap_locker(m_image_ctx.snap_lock); + RWLock::WLocker parent_locker(m_image_ctx.parent_lock); + + m_image_ctx.size = m_size; + m_image_ctx.lockers = m_lockers; + m_image_ctx.lock_tag = m_lock_tag; + m_image_ctx.exclusive_locked = m_exclusive_locked; + + if (m_image_ctx.old_format) { + m_image_ctx.order = m_order; + m_image_ctx.features = 0; + m_image_ctx.flags = 0; + m_image_ctx.object_prefix = std::move(m_object_prefix); + m_image_ctx.init_layout(); + } else { + m_image_ctx.features = m_features; + m_image_ctx.flags = m_flags; + m_image_ctx.parent_md = m_parent_md; + } + + for (size_t i = 0; i < m_snapc.snaps.size(); ++i) { + std::vector::const_iterator it = std::find( + m_image_ctx.snaps.begin(), m_image_ctx.snaps.end(), + m_snapc.snaps[i].val); + if (it == m_image_ctx.snaps.end()) { + m_flush_aio = true; + ldout(cct, 20) << "new snapshot id=" << m_snapc.snaps[i].val + << " name=" << m_snap_names[i] + << " size=" << m_snap_sizes[i] + << dendl; + } + } + + m_image_ctx.snaps.clear(); + m_image_ctx.snap_info.clear(); + m_image_ctx.snap_ids.clear(); + for (size_t i = 0; i < m_snapc.snaps.size(); ++i) { + uint64_t flags = m_image_ctx.old_format ? 0 : m_snap_flags[i]; + uint8_t protection_status = m_image_ctx.old_format ? + static_cast(RBD_PROTECTION_STATUS_UNPROTECTED) : + m_snap_protection[i]; + parent_info parent; + if (!m_image_ctx.old_format) { + parent = m_snap_parents[i]; + } + + m_image_ctx.add_snap(m_snap_names[i], m_snapc.snaps[i].val, + m_snap_sizes[i], parent, protection_status, flags); + } + m_image_ctx.snapc = m_snapc; + + if (m_image_ctx.snap_id != CEPH_NOSNAP && + m_image_ctx.get_snap_id(m_image_ctx.snap_name) != m_image_ctx.snap_id) { + lderr(cct) << "tried to read from a snapshot that no longer exists: " + << m_image_ctx.snap_name << dendl; + m_image_ctx.snap_exists = false; + } + + if (m_refresh_parent != nullptr) { + m_refresh_parent->apply(); + } + m_image_ctx.data_ctx.selfmanaged_snap_set_write_ctx(m_image_ctx.snapc.seq, + m_image_ctx.snaps); + + // handle dynamically enabled / disabled features + if (!m_image_ctx.test_features(RBD_FEATURE_EXCLUSIVE_LOCK, + m_image_ctx.snap_lock) || + m_exclusive_lock != nullptr) { + std::swap(m_exclusive_lock, m_image_ctx.exclusive_lock); + } + if (!m_image_ctx.test_features(RBD_FEATURE_JOURNALING, + m_image_ctx.snap_lock) || + m_journal != nullptr) { + std::swap(m_journal, m_image_ctx.journal); + } + if (!m_image_ctx.test_features(RBD_FEATURE_OBJECT_MAP, + m_image_ctx.snap_lock) || + m_object_map != nullptr) { + std::swap(m_object_map, m_image_ctx.object_map); + } + } +} + +template +int RefreshRequest::get_parent_info(uint64_t snap_id, + parent_info *parent_md) { + if (snap_id == CEPH_NOSNAP) { + *parent_md = m_parent_md; + return 0; + } else { + for (size_t i = 0; i < m_snapc.snaps.size(); ++i) { + if (m_snapc.snaps[i].val == snap_id) { + *parent_md = m_snap_parents[i]; + return 0; + } + } + } + return -ENOENT; +} + +} // namespace image +} // namespace librbd + +template class librbd::image::RefreshRequest; diff --git a/src/librbd/image/RefreshRequest.h b/src/librbd/image/RefreshRequest.h new file mode 100644 index 0000000000000..25eda668a2fe3 --- /dev/null +++ b/src/librbd/image/RefreshRequest.h @@ -0,0 +1,189 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IMAGE_REFRESH_REQUEST_H +#define CEPH_LIBRBD_IMAGE_REFRESH_REQUEST_H + +#include "include/int_types.h" +#include "include/buffer.h" +#include "include/rbd_types.h" +#include "common/snap_types.h" +#include "cls/lock/cls_lock_types.h" +#include "librbd/ImageCtx.h" +#include "librbd/parent_types.h" +#include +#include + +class Context; + +namespace librbd { + +template class ExclusiveLock; +class ImageCtx; +class Journal; +class ObjectMap; + +namespace image { + +template class RefreshParentRequest; + +template +class RefreshRequest { +public: + static RefreshRequest *create(ImageCtxT &image_ctx, Context *on_finish) { + return new RefreshRequest(image_ctx, on_finish); + } + + ~RefreshRequest(); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * | (v1) + * |-----> V1_READ_HEADER ---> V1_GET_SNAPSHOTS ---> V1_GET_LOCKS + * | | + * | (v2) v + * \-----> V2_GET_MUTABLE_METADATA + * | | + * v | + * V2_GET_FLAGS | + * | | + * v | + * V2_GET_SNAPSHOTS (skip if no snaps) | + * | | + * v | + * V2_REFRESH_PARENT (skip if no parent or | + * | refresh not needed) | + * v | + * V2_INIT_EXCLUSIVE_LOCK (skip if lock | + * | active or disabled) | + * v | + * V2_OPEN_JOURNAL (skip if journal | + * | active or disabled) | + * v | + * V2_OPEN_OBJECT_MAP (skip if map | + * | active or disabled) | + * v | + * | + * | | + * v | + * V2_FINALIZE_REFRESH_PARENT (skip if refresh | + * | not needed) | + * (error) v | + * * * * * > V2_SHUT_DOWN_EXCLUSIVE_LOCK (skip if lock | + * | active or enabled) | + * v | + * V2_CLOSE_JOURNAL (skip if journal inactive | + * | or enabled) | + * v | + * V2_CLOSE_OBJECT_MAP (skip if map inactive | + * | or enabled) | + * | | + * \-------------------\/--------------------/ + * | + * v + * FLUSH (skip if no new + * | snapshots) + * v + * + * + * @endverbatim + */ + + ImageCtxT &m_image_ctx; + Context *m_on_finish; + + int m_error_result; + bool m_flush_aio; + decltype(m_image_ctx.exclusive_lock) m_exclusive_lock; + decltype(m_image_ctx.object_map) m_object_map; + decltype(m_image_ctx.journal) m_journal; + RefreshParentRequest *m_refresh_parent; + + bufferlist m_out_bl; + + uint8_t m_order; + uint64_t m_size; + uint64_t m_features; + uint64_t m_incompatible_features; + uint64_t m_flags; + std::string m_object_prefix; + parent_info m_parent_md; + + ::SnapContext m_snapc; + std::vector m_snap_names; + std::vector m_snap_sizes; + std::vector m_snap_parents; + std::vector m_snap_protection; + std::vector m_snap_flags; + + std::map m_lockers; + std::string m_lock_tag; + bool m_exclusive_locked; + + RefreshRequest(ImageCtxT &image_ctx, Context *on_finish); + + void send_v1_read_header(); + Context *handle_v1_read_header(int *result); + + void send_v1_get_snapshots(); + Context *handle_v1_get_snapshots(int *result); + + void send_v1_get_locks(); + Context *handle_v1_get_locks(int *result); + + void send_v2_get_mutable_metadata(); + Context *handle_v2_get_mutable_metadata(int *result); + + void send_v2_get_flags(); + Context *handle_v2_get_flags(int *result); + + Context *send_v2_get_snapshots(); + Context *handle_v2_get_snapshots(int *result); + + Context *send_v2_refresh_parent(); + Context *handle_v2_refresh_parent(int *result); + + Context *send_v2_init_exclusive_lock(); + Context *handle_v2_init_exclusive_lock(int *result); + + Context *send_v2_open_journal(); + Context *handle_v2_open_journal(int *result); + + Context *send_v2_open_object_map(); + Context *handle_v2_open_object_map(int *result); + + Context *send_v2_finalize_refresh_parent(); + Context *handle_v2_finalize_refresh_parent(int *result); + + Context *send_v2_shut_down_exclusive_lock(); + Context *handle_v2_shut_down_exclusive_lock(int *result); + + Context *send_v2_close_journal(); + Context *handle_v2_close_journal(int *result); + + Context *send_flush_aio(); + Context *handle_flush_aio(int *result); + + void save_result(int *result) { + if (m_error_result == 0 && *result < 0) { + m_error_result = *result; + } + } + + void apply(); + int get_parent_info(uint64_t snap_id, parent_info *parent_md); +}; + +} // namespace image +} // namespace librbd + +extern template class librbd::image::RefreshRequest; + +#endif // CEPH_LIBRBD_IMAGE_REFRESH_REQUEST_H diff --git a/src/librbd/image/SetSnapRequest.cc b/src/librbd/image/SetSnapRequest.cc new file mode 100644 index 0000000000000..d88545e854fe1 --- /dev/null +++ b/src/librbd/image/SetSnapRequest.cc @@ -0,0 +1,342 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/image/SetSnapRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/AioImageRequestWQ.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ObjectMap.h" +#include "librbd/Utils.h" +#include "librbd/image/RefreshParentRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::image::SetSnapRequest: " + +namespace librbd { +namespace image { + +using util::create_context_callback; + +template +SetSnapRequest::SetSnapRequest(I &image_ctx, const std::string &snap_name, + Context *on_finish) + : m_image_ctx(image_ctx), m_snap_name(snap_name), m_on_finish(on_finish), + m_snap_id(CEPH_NOSNAP), m_exclusive_lock(nullptr), m_object_map(nullptr), + m_refresh_parent(nullptr), m_writes_blocked(false) { +} + +template +SetSnapRequest::~SetSnapRequest() { + delete m_refresh_parent; + delete m_object_map; + delete m_exclusive_lock; + if (m_writes_blocked) { + m_image_ctx.aio_work_queue->unblock_writes(); + } +} + +template +void SetSnapRequest::send() { + if (m_snap_name.empty()) { + send_init_exclusive_lock(); + } else { + send_block_writes(); + } +} + +template +void SetSnapRequest::send_init_exclusive_lock() { + { + RWLock::RLocker snap_locker(m_image_ctx.snap_lock); + if (m_image_ctx.exclusive_lock != nullptr) { + assert(m_image_ctx.snap_id == CEPH_NOSNAP); + send_complete(); + return; + } + } + + if (!m_image_ctx.test_features(RBD_FEATURE_EXCLUSIVE_LOCK)) { + int r = 0; + if (send_refresh_parent(&r) != nullptr) { + send_complete(); + return; + } + } + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << __func__ << dendl; + + m_exclusive_lock = ExclusiveLock::create(m_image_ctx); + + using klass = SetSnapRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_init_exclusive_lock>(this); + + RWLock::RLocker owner_locker(m_image_ctx.owner_lock); + m_exclusive_lock->init(ctx); +} + +template +Context *SetSnapRequest::handle_init_exclusive_lock(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to initialize exclusive lock: " + << cpp_strerror(*result) << dendl; + return m_on_finish; + } + return send_refresh_parent(result); +} + +template +void SetSnapRequest::send_block_writes() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << __func__ << dendl; + + m_writes_blocked = true; + + using klass = SetSnapRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_block_writes>(this); + + RWLock::RLocker owner_locker(m_image_ctx.owner_lock); + m_image_ctx.aio_work_queue->block_writes(ctx); +} + +template +Context *SetSnapRequest::handle_block_writes(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to block writes: " << cpp_strerror(*result) + << dendl; + return m_on_finish; + } + + { + RWLock::RLocker snap_locker(m_image_ctx.snap_lock); + m_snap_id = m_image_ctx.get_snap_id(m_snap_name); + if (m_snap_id == CEPH_NOSNAP) { + lderr(cct) << "failed to locate snapshot '" << m_snap_name << "'" + << dendl; + + *result = -ENOENT; + return m_on_finish; + } + } + + return send_shut_down_exclusive_lock(result); +} + +template +Context *SetSnapRequest::send_shut_down_exclusive_lock(int *result) { + ExclusiveLock *exclusive_lock; + { + RWLock::RLocker snap_locker(m_image_ctx.snap_lock); + exclusive_lock = m_image_ctx.exclusive_lock; + } + + if (exclusive_lock == nullptr) { + return send_refresh_parent(result); + } + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << __func__ << dendl; + + using klass = SetSnapRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_shut_down_exclusive_lock>(this); + exclusive_lock->shut_down(ctx); + return nullptr; +} + +template +Context *SetSnapRequest::handle_shut_down_exclusive_lock(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to shut down exclusive lock: " + << cpp_strerror(*result) << dendl; + return m_on_finish; + } + + return send_refresh_parent(result); +} + +template +Context *SetSnapRequest::send_refresh_parent(int *result) { + CephContext *cct = m_image_ctx.cct; + + parent_info parent_md; + bool refresh_parent; + { + RWLock::RLocker snap_locker(m_image_ctx.snap_lock); + RWLock::RLocker parent_locker(m_image_ctx.parent_lock); + + const parent_info *parent_info = m_image_ctx.get_parent_info(m_snap_id); + if (parent_info == nullptr) { + *result = -ENOENT; + lderr(cct) << "failed to retrieve snapshot parent info" << dendl; + return m_on_finish; + } + + parent_md = *parent_info; + refresh_parent = RefreshParentRequest::is_refresh_required(m_image_ctx, + parent_md); + } + + if (!refresh_parent) { + if (m_snap_id == CEPH_NOSNAP) { + // object map is loaded when exclusive lock is acquired + *result = apply(); + return m_on_finish; + } else { + // load snapshot object map + return send_open_object_map(result); + } + } + + ldout(cct, 10) << __func__ << dendl; + + using klass = SetSnapRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_refresh_parent>(this); + m_refresh_parent = RefreshParentRequest::create(m_image_ctx, parent_md, + ctx); + m_refresh_parent->send(); + return nullptr; +} + +template +Context *SetSnapRequest::handle_refresh_parent(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to refresh snapshot parent: " << cpp_strerror(*result) + << dendl; + return m_on_finish; + } + + if (m_snap_id == CEPH_NOSNAP) { + // object map is loaded when exclusive lock is acquired + *result = apply(); + if (*result < 0) { + return m_on_finish; + } + + return send_finalize_refresh_parent(result); + } else { + // load snapshot object map + return send_open_object_map(result); + } +} + +template +Context *SetSnapRequest::send_open_object_map(int *result) { + if (!m_image_ctx.test_features(RBD_FEATURE_OBJECT_MAP)) { + *result = apply(); + if (*result < 0) { + return m_on_finish; + } + + return send_finalize_refresh_parent(result); + } + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << __func__ << dendl; + + using klass = SetSnapRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_open_object_map>(this); + m_object_map = new ObjectMap(m_image_ctx, m_snap_id); + m_object_map->open(ctx); + return nullptr; +} + +template +Context *SetSnapRequest::handle_open_object_map(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << __func__ << ": r=" << *result << dendl; + + // object map should never report errors + assert(*result == 0); + + *result = apply(); + if (*result < 0) { + return m_on_finish; + } + + return send_finalize_refresh_parent(result); +} + +template +Context *SetSnapRequest::send_finalize_refresh_parent(int *result) { + if (m_refresh_parent == nullptr) { + return m_on_finish; + } + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + using klass = SetSnapRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_finalize_refresh_parent>(this); + m_refresh_parent->finalize(ctx); + return nullptr; +} + +template +Context *SetSnapRequest::handle_finalize_refresh_parent(int *result) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to close parent image: " << cpp_strerror(*result) + << dendl; + } + return m_on_finish; +} + +template +int SetSnapRequest::apply() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << __func__ << dendl; + + RWLock::WLocker snap_locker(m_image_ctx.snap_lock); + RWLock::WLocker parent_locker(m_image_ctx.parent_lock); + + if (m_snap_id != CEPH_NOSNAP) { + int r = m_image_ctx.snap_set(m_snap_name); + if (r < 0) { + return r; + } + } else { + m_image_ctx.snap_unset(); + } + + if (m_refresh_parent != nullptr) { + m_refresh_parent->apply(); + } + + std::swap(m_exclusive_lock, m_image_ctx.exclusive_lock); + std::swap(m_object_map, m_image_ctx.object_map_ptr); + return 0; +} + +template +void SetSnapRequest::send_complete() { + m_on_finish->complete(0); + delete this; +} + +} // namespace image +} // namespace librbd + +template class librbd::image::SetSnapRequest; diff --git a/src/librbd/image/SetSnapRequest.h b/src/librbd/image/SetSnapRequest.h new file mode 100644 index 0000000000000..815614ead5f06 --- /dev/null +++ b/src/librbd/image/SetSnapRequest.h @@ -0,0 +1,121 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IMAGE_SNAP_SET_REQUEST_H +#define CEPH_LIBRBD_IMAGE_SNAP_SET_REQUEST_H + +#include "include/int_types.h" +#include "librbd/parent_types.h" +#include + +class Context; + +namespace librbd { + +template class ExclusiveLock; +class ImageCtx; +class ObjectMap; + +namespace image { + +template class RefreshParentRequest; + +template +class SetSnapRequest { +public: + static SetSnapRequest *create(ImageCtxT &image_ctx, + const std::string &snap_name, + Context *on_finish) { + return new SetSnapRequest(image_ctx, snap_name, on_finish); + } + + ~SetSnapRequest(); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * | (set snap) + * |-----------> BLOCK_WRITES + * | | + * | v + * | SHUTDOWN_EXCLUSIVE_LOCK (skip if lock inactive + * | | or disabled) + * | v + * | REFRESH_PARENT (skip if no parent + * | | or refresh not needed) + * | v + * | OPEN_OBJECT_MAP (skip if map disabled) + * | | + * | v + * | + * | | + * | v + * | FINALIZE_REFRESH_PARENT (skip if no parent + * | | or refresh not needed) + * | v + * | + * | + * \-----------> INIT_EXCLUSIVE_LOCK (skip if active or + * | disabled) + * v + * REFRESH_PARENT (skip if no parent + * | or refresh not needed) + * v + * + * | + * v + * FINALIZE_REFRESH_PARENT (skip if no parent + * | or refresh not needed) + * v + * + * + * @endverbatim + */ + + SetSnapRequest(ImageCtxT &image_ctx, const std::string &snap_name, + Context *on_finish); + + ImageCtxT &m_image_ctx; + std::string m_snap_name; + Context *m_on_finish; + + uint64_t m_snap_id; + ExclusiveLock *m_exclusive_lock; + ObjectMap *m_object_map; + RefreshParentRequest *m_refresh_parent; + + bool m_writes_blocked; + + void send_block_writes(); + Context *handle_block_writes(int *result); + + void send_init_exclusive_lock(); + Context *handle_init_exclusive_lock(int *result); + + Context *send_shut_down_exclusive_lock(int *result); + Context *handle_shut_down_exclusive_lock(int *result); + + Context *send_refresh_parent(int *result); + Context *handle_refresh_parent(int *result); + + Context *send_open_object_map(int *result); + Context *handle_open_object_map(int *result); + + Context *send_finalize_refresh_parent(int *result); + Context *handle_finalize_refresh_parent(int *result); + + int apply(); + void send_complete(); +}; + +} // namespace image +} // namespace librbd + +extern template class librbd::image::SetSnapRequest; + +#endif // CEPH_LIBRBD_IMAGE_SNAP_SET_REQUEST_H diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index 24150d94d58fd..b5e77e73f498a 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -31,6 +31,12 @@ #include "librbd/Journal.h" #include "librbd/ObjectMap.h" #include "librbd/parent_types.h" +#include "librbd/Utils.h" +#include "librbd/image/CloseRequest.h" +#include "librbd/image/OpenRequest.h" +#include "librbd/image/RefreshParentRequest.h" +#include "librbd/image/RefreshRequest.h" +#include "librbd/image/SetSnapRequest.h" #include "librbd/operation/FlattenRequest.h" #include "librbd/operation/RebuildObjectMapRequest.h" #include "librbd/operation/RenameRequest.h" @@ -42,7 +48,6 @@ #include "librbd/operation/SnapshotRollbackRequest.h" #include "librbd/operation/SnapshotUnprotectRequest.h" #include "librbd/operation/TrimRequest.h" -#include "librbd/Utils.h" #include "include/util.h" #include @@ -2959,7 +2964,7 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) { << "' id = '" << ictx->id << "' snap_name = '" << ictx->snap_name << "'" << dendl; - int r = ictx->init(); + int r = ictx->init_legacy(); if (r < 0) goto err_close; -- 2.39.5