From: Jason Dillaman Date: Tue, 16 Feb 2016 01:49:22 +0000 (-0500) Subject: librbd: support for AIO notifications X-Git-Tag: v10.1.0~357^2~10 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=dd5c9e55031b9791a1c53ce00a9ac49c55d17cae;p=ceph.git librbd: support for AIO notifications The header update and lock notifications might be invoked from the librados AIO thread. Update the close state machine to flush any potential AIO notifications. Signed-off-by: Jason Dillaman --- diff --git a/src/librbd/ImageWatcher.cc b/src/librbd/ImageWatcher.cc index 76ca7c558bdd..66e9a0b0daa5 100644 --- a/src/librbd/ImageWatcher.cc +++ b/src/librbd/ImageWatcher.cc @@ -10,6 +10,7 @@ #include "librbd/Operations.h" #include "librbd/TaskFinisher.h" #include "librbd/Utils.h" +#include "librbd/image_watcher/Notifier.h" #include "include/encoding.h" #include "include/stringify.h" #include "common/errno.h" @@ -24,9 +25,9 @@ namespace librbd { +using namespace image_watcher; using namespace watch_notify; -static const uint64_t NOTIFY_TIMEOUT = 5000; static const double RETRY_DELAY_SECONDS = 1.0; ImageWatcher::ImageWatcher(ImageCtx &image_ctx) @@ -35,7 +36,8 @@ ImageWatcher::ImageWatcher(ImageCtx &image_ctx) m_watch_ctx(*this), m_watch_handle(0), m_watch_state(WATCH_STATE_UNREGISTERED), m_async_request_lock(util::unique_lock_name("librbd::ImageWatcher::m_async_request_lock", this)), - m_owner_client_id_lock(util::unique_lock_name("librbd::ImageWatcher::m_owner_client_id_lock", this)) + m_owner_client_id_lock(util::unique_lock_name("librbd::ImageWatcher::m_owner_client_id_lock", this)), + m_notifier(image_ctx) { m_image_ctx.cct->lookup_or_create_singleton_object >( m_task_finisher, "librbd::task_finisher"); @@ -86,6 +88,10 @@ int ImageWatcher::unregister_watch() { return r; } +void ImageWatcher::flush(Context *on_finish) { + m_notifier.flush(on_finish); +} + void ImageWatcher::schedule_async_progress(const AsyncRequestId &request, uint64_t offset, uint64_t total) { FunctionContext *ctx = new FunctionContext( @@ -127,7 +133,7 @@ int ImageWatcher::notify_async_complete(const AsyncRequestId &request, &m_image_ctx); } int ret = m_image_ctx.md_ctx.notify2(m_image_ctx.header_oid, bl, - NOTIFY_TIMEOUT, NULL); + Notifier::NOTIFY_TIMEOUT, NULL); if (ret < 0) { lderr(m_image_ctx.cct) << this << " failed to notify async complete: " << cpp_strerror(ret) << dendl; @@ -393,8 +399,8 @@ int ImageWatcher::notify_lock_owner(bufferlist &bl) { // case another notification occurs before this one and it requires the lock bufferlist response_bl; m_image_ctx.owner_lock.put_read(); - int r = m_image_ctx.md_ctx.notify2(m_image_ctx.header_oid, bl, NOTIFY_TIMEOUT, - &response_bl); + int r = m_image_ctx.md_ctx.notify2(m_image_ctx.header_oid, bl, + Notifier::NOTIFY_TIMEOUT, &response_bl); m_image_ctx.owner_lock.get_read(); if (r < 0 && r != -ETIMEDOUT) { diff --git a/src/librbd/ImageWatcher.h b/src/librbd/ImageWatcher.h index 847b27e30a44..873ba34ca49e 100644 --- a/src/librbd/ImageWatcher.h +++ b/src/librbd/ImageWatcher.h @@ -3,12 +3,12 @@ #ifndef CEPH_LIBRBD_IMAGE_WATCHER_H #define CEPH_LIBRBD_IMAGE_WATCHER_H -#include "common/Cond.h" #include "common/Mutex.h" #include "common/RWLock.h" #include "include/Context.h" #include "include/rados/librados.hpp" #include "include/rbd/librbd.hpp" +#include "librbd/image_watcher/Notifier.h" #include "librbd/WatchNotifyTypes.h" #include #include @@ -31,6 +31,7 @@ public: int register_watch(); int unregister_watch(); + void flush(Context *on_finish); int notify_flatten(uint64_t request_id, ProgressContext &prog_ctx); int notify_resize(uint64_t request_id, uint64_t size, @@ -65,9 +66,7 @@ private: }; enum TaskCode { - TASK_CODE_ACQUIRED_LOCK, TASK_CODE_REQUEST_LOCK, - TASK_CODE_RELEASED_LOCK, TASK_CODE_CANCEL_ASYNC_REQUESTS, TASK_CODE_REREGISTER_WATCH, TASK_CODE_ASYNC_REQUEST, @@ -223,6 +222,8 @@ private: Mutex m_owner_client_id_lock; watch_notify::ClientId m_owner_client_id; + image_watcher::Notifier m_notifier; + void schedule_cancel_async_requests(); void cancel_async_requests(); diff --git a/src/librbd/Makefile.am b/src/librbd/Makefile.am index 7248f82cfeed..22a191ab76a5 100644 --- a/src/librbd/Makefile.am +++ b/src/librbd/Makefile.am @@ -35,6 +35,7 @@ librbd_internal_la_SOURCES = \ librbd/image/RefreshParentRequest.cc \ librbd/image/RefreshRequest.cc \ librbd/image/SetSnapRequest.cc \ + librbd/image_watcher/Notifier.cc \ librbd/journal/Replay.cc \ librbd/object_map/InvalidateRequest.cc \ librbd/object_map/LockRequest.cc \ @@ -114,6 +115,7 @@ noinst_HEADERS += \ librbd/image/RefreshParentRequest.h \ librbd/image/RefreshRequest.h \ librbd/image/SetSnapRequest.h \ + librbd/image_watcher/Notifier.h \ librbd/journal/Replay.h \ librbd/journal/Types.h \ librbd/object_map/InvalidateRequest.h \ diff --git a/src/librbd/Utils.cc b/src/librbd/Utils.cc index d89418f490cd..3d73830baa0f 100644 --- a/src/librbd/Utils.cc +++ b/src/librbd/Utils.cc @@ -27,5 +27,9 @@ std::string unique_lock_name(const std::string &name, void *address) { return name + " (" + stringify(address) + ")"; } +librados::AioCompletion *create_rados_ack_callback(Context *on_finish) { + return create_rados_ack_callback(on_finish); +} + } // namespace util } // namespace librbd diff --git a/src/librbd/Utils.h b/src/librbd/Utils.h index 0986f060b981..1287e597d662 100644 --- a/src/librbd/Utils.h +++ b/src/librbd/Utils.h @@ -23,6 +23,13 @@ void rados_callback(rados_completion_t c, void *arg) { reinterpret_cast(arg)->complete(rados_aio_get_return_value(c)); } +template +void rados_callback(rados_completion_t c, void *arg) { + T *obj = reinterpret_cast(arg); + int r = rados_aio_get_return_value(c); + (obj->*MF)(r); +} + template void rados_state_callback(rados_completion_t c, void *arg) { T *obj = reinterpret_cast(arg); @@ -91,12 +98,20 @@ const std::string header_name(const std::string &image_id); const std::string old_header_name(const std::string &image_name); std::string unique_lock_name(const std::string &name, void *address); +librados::AioCompletion *create_rados_ack_callback(Context *on_finish); + template librados::AioCompletion *create_rados_ack_callback(T *obj) { return librados::Rados::aio_create_completion( obj, &detail::rados_callback, nullptr); } +template +librados::AioCompletion *create_rados_ack_callback(T *obj) { + return librados::Rados::aio_create_completion( + obj, &detail::rados_callback, nullptr); +} + template librados::AioCompletion *create_rados_ack_callback(T *obj) { return librados::Rados::aio_create_completion( diff --git a/src/librbd/image/CloseRequest.cc b/src/librbd/image/CloseRequest.cc index 57f04a7d8eca..145a998dbf6c 100644 --- a/src/librbd/image/CloseRequest.cc +++ b/src/librbd/image/CloseRequest.cc @@ -21,7 +21,6 @@ namespace librbd { namespace image { - using util::create_async_context_callback; using util::create_context_callback; @@ -202,7 +201,7 @@ void CloseRequest::handle_flush_op_work_queue(int r) { template void CloseRequest::send_close_parent() { if (m_image_ctx->parent == nullptr) { - finish(); + send_flush_image_watcher(); return; } @@ -224,12 +223,32 @@ void CloseRequest::handle_close_parent(int r) { if (r < 0) { lderr(cct) << "error closing parent image: " << cpp_strerror(r) << dendl; } + send_flush_image_watcher(); +} + +template +void CloseRequest::send_flush_image_watcher() { + if (m_image_ctx->image_watcher == nullptr) { + finish(); + return; + } + + m_image_ctx->image_watcher->flush(create_context_callback< + CloseRequest, &CloseRequest::handle_flush_image_watcher>(this)); +} + +template +void CloseRequest::handle_flush_image_watcher(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + assert(r == 0); finish(); } template void CloseRequest::finish() { - if (m_image_ctx->image_watcher) { + if (m_image_ctx->image_watcher != nullptr) { m_image_ctx->unregister_watch(); } diff --git a/src/librbd/image/CloseRequest.h b/src/librbd/image/CloseRequest.h index 08ace956752e..832cd2e1af54 100644 --- a/src/librbd/image/CloseRequest.h +++ b/src/librbd/image/CloseRequest.h @@ -49,13 +49,16 @@ private: * SHUTDOWN_CACHE * | * v - * FLUSH_OP_WORK_QUEUE . . . . . - * | . - * v . - * CLOSE_PARENT . (no parent) - * | . - * v . - * < . . . . . . . . . . + * FLUSH_OP_WORK_QUEUE . . . . . + * | . + * v . + * CLOSE_PARENT . (no parent) + * | . + * v . + * FLUSH_IMAGE_WATCHER < . . . . + * | + * v + * * * @endverbatim */ @@ -93,6 +96,9 @@ private: void send_close_parent(); void handle_close_parent(int r); + void send_flush_image_watcher(); + void handle_flush_image_watcher(int r); + void finish(); void save_result(int result) { diff --git a/src/librbd/image_watcher/Notifier.cc b/src/librbd/image_watcher/Notifier.cc new file mode 100644 index 000000000000..29cd56f787a2 --- /dev/null +++ b/src/librbd/image_watcher/Notifier.cc @@ -0,0 +1,76 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/image_watcher/Notifier.h" +#include "common/WorkQueue.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::image_watcher::Notifier: " + +namespace librbd { +namespace image_watcher { + +const uint64_t Notifier::NOTIFY_TIMEOUT = 5000; + +Notifier::Notifier(ImageCtx &image_ctx) + : m_image_ctx(image_ctx), + m_aio_notify_lock(util::unique_lock_name( + "librbd::image_watcher::Notifier::m_aio_notify_lock", this)) { +} + +Notifier::~Notifier() { + Mutex::Locker aio_notify_locker(m_aio_notify_lock); + assert(m_pending_aio_notifies == 0); +} + +void Notifier::flush(Context *on_finish) { + Mutex::Locker aio_notify_locker(m_aio_notify_lock); + if (m_pending_aio_notifies == 0) { + m_image_ctx.op_work_queue->queue(on_finish, 0); + return; + } + + assert(m_aio_notify_flush == nullptr); + m_aio_notify_flush = on_finish; +} + +void Notifier::notify(bufferlist &bl, bufferlist *out_bl, Context *on_finish) { + { + Mutex::Locker aio_notify_locker(m_aio_notify_lock); + ++m_pending_aio_notifies; + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << __func__ << ": pending=" << m_pending_aio_notifies + << dendl; + } + + C_AioNotify *ctx = new C_AioNotify(this, on_finish); + librados::AioCompletion *comp = util::create_rados_ack_callback(ctx); + int r = m_image_ctx.md_ctx.aio_notify(m_image_ctx.header_oid, comp, bl, + NOTIFY_TIMEOUT, nullptr); + assert(r == 0); + comp->release(); +} + +void Notifier::handle_notify(int r, Context *on_finish) { + if (on_finish != nullptr) { + m_image_ctx.op_work_queue->queue(on_finish, r); + } + + Mutex::Locker aio_notify_locker(m_aio_notify_lock); + assert(m_pending_aio_notifies > 0); + --m_pending_aio_notifies; + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << __func__ << ": pending=" << m_pending_aio_notifies + << dendl; + if (m_pending_aio_notifies == 0 && m_aio_notify_flush != nullptr) { + m_image_ctx.op_work_queue->queue(m_aio_notify_flush, 0); + } +} + +} // namespace image_watcher +} // namespace librbd diff --git a/src/librbd/image_watcher/Notifier.h b/src/librbd/image_watcher/Notifier.h new file mode 100644 index 000000000000..008f6cbfe9a3 --- /dev/null +++ b/src/librbd/image_watcher/Notifier.h @@ -0,0 +1,54 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IMAGE_WATCHER_NOTIFIER_H +#define CEPH_LIBRBD_IMAGE_WATCHER_NOTIFIER_H + +#include "include/int_types.h" +#include "include/buffer.h" +#include "include/Context.h" +#include "common/Mutex.h" + +namespace librbd { + +struct ImageCtx; + +namespace image_watcher { + +class Notifier { +public: + static const uint64_t NOTIFY_TIMEOUT; + + Notifier(ImageCtx &image_ctx); + ~Notifier(); + + void flush(Context *on_finish); + void notify(bufferlist &bl, bufferlist *out_bl, Context *on_finish); + +private: + struct C_AioNotify : public Context { + Notifier *notifier; + Context *on_finish; + + C_AioNotify(Notifier *notifier, Context *on_finish) + : notifier(notifier), on_finish(on_finish) { + } + virtual void finish(int r) override { + notifier->handle_notify(r, on_finish); + } + }; + + ImageCtx &m_image_ctx; + + Mutex m_aio_notify_lock; + size_t m_pending_aio_notifies = 0; + Context *m_aio_notify_flush = nullptr; + + void handle_notify(int r, Context *on_finish); + +}; + +} // namespace image_watcher +} // namespace librbd + +#endif // CEPH_LIBRBD_IMAGE_WATCHER_NOTIFIER_H