From 004090a66062a3cee6750145178c9660ad04dced Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 11 Nov 2015 21:34:24 -0500 Subject: [PATCH] librbd: cleanup librbd AIO completion callbacks Signed-off-by: Jason Dillaman --- src/librbd/AioCompletion.h | 24 ++++++++++++++++++ src/librbd/AioImageRequestWQ.cc | 6 ++--- src/librbd/AioObjectRequest.cc | 10 +------- src/librbd/AioObjectRequest.h | 3 ++- src/librbd/CopyupRequest.cc | 3 +-- src/librbd/JournalReplay.cc | 4 +-- src/librbd/internal.cc | 45 ++++++++------------------------- src/librbd/internal.h | 8 ------ src/librbd/librbd.cc | 6 ++--- 9 files changed, 45 insertions(+), 64 deletions(-) diff --git a/src/librbd/AioCompletion.h b/src/librbd/AioCompletion.h index 6bbe34a0649..0d1e226061b 100644 --- a/src/librbd/AioCompletion.h +++ b/src/librbd/AioCompletion.h @@ -68,6 +68,30 @@ namespace librbd { xlist::item m_xlist_item; bool event_notify; + template + static void callback_adapter(completion_t cb, void *arg) { + AioCompletion *comp = reinterpret_cast(cb); + T *t = reinterpret_cast(arg); + (t->*MF)(comp->get_return_value()); + comp->release(); + } + + static AioCompletion *create(void *cb_arg, callback_t cb_complete, + rbd_completion_t rbd_comp) { + AioCompletion *comp = new AioCompletion(); + comp->set_complete_cb(cb_arg, cb_complete); + comp->rbd_comp = (rbd_comp != nullptr ? rbd_comp : comp); + return comp; + } + + template + static AioCompletion *create(T *obj) { + AioCompletion *comp = new AioCompletion(); + comp->set_complete_cb(obj, &callback_adapter); + comp->rbd_comp = comp; + return comp; + } + AioCompletion() : lock("AioCompletion::lock", true, false), done(false), rval(0), complete_cb(NULL), complete_arg(NULL), rbd_comp(NULL), diff --git a/src/librbd/AioImageRequestWQ.cc b/src/librbd/AioImageRequestWQ.cc index d4690f1aa23..8dbfd755536 100644 --- a/src/librbd/AioImageRequestWQ.cc +++ b/src/librbd/AioImageRequestWQ.cc @@ -34,7 +34,7 @@ ssize_t AioImageRequestWQ::read(uint64_t off, uint64_t len, char *buf, image_extents.push_back(make_pair(off, len)); C_SaferCond cond; - AioCompletion *c = aio_create_completion_internal(&cond, rbd_ctx_cb); + AioCompletion *c = AioCompletion::create(&cond); aio_read(c, off, len, buf, NULL, op_flags, false); return cond.wait(); } @@ -53,7 +53,7 @@ ssize_t AioImageRequestWQ::write(uint64_t off, uint64_t len, const char *buf, } C_SaferCond cond; - AioCompletion *c = aio_create_completion_internal(&cond, rbd_ctx_cb); + AioCompletion *c = AioCompletion::create(&cond); aio_write(c, off, len, buf, op_flags, false); r = cond.wait(); @@ -76,7 +76,7 @@ int AioImageRequestWQ::discard(uint64_t off, uint64_t len) { } C_SaferCond cond; - AioCompletion *c = aio_create_completion_internal(&cond, rbd_ctx_cb); + AioCompletion *c = AioCompletion::create(&cond); aio_discard(c, off, len, false); r = cond.wait(); diff --git a/src/librbd/AioObjectRequest.cc b/src/librbd/AioObjectRequest.cc index aef2f181473..aef9870de96 100644 --- a/src/librbd/AioObjectRequest.cc +++ b/src/librbd/AioObjectRequest.cc @@ -103,14 +103,6 @@ namespace librbd { guard_read(); } - AioObjectRead::~AioObjectRead() - { - if (m_parent_completion) { - m_parent_completion->release(); - m_parent_completion = NULL; - } - } - void AioObjectRead::guard_read() { RWLock::RLocker snap_locker(m_ictx->snap_lock); @@ -262,7 +254,7 @@ namespace librbd { void AioObjectRead::read_from_parent(const vector >& parent_extents) { assert(!m_parent_completion); - m_parent_completion = aio_create_completion_internal(this, rbd_req_cb); + m_parent_completion = AioCompletion::create(this); // prevent the parent image from being deleted while this // request is still in-progress diff --git a/src/librbd/AioObjectRequest.h b/src/librbd/AioObjectRequest.h index bcbaf6b50ae..7aa06c1d5c5 100644 --- a/src/librbd/AioObjectRequest.h +++ b/src/librbd/AioObjectRequest.h @@ -11,6 +11,7 @@ #include "include/buffer.h" #include "include/Context.h" #include "include/rados/librados.hpp" +#include "include/rbd/librbd.hpp" #include "librbd/ObjectMap.h" namespace librbd { @@ -63,7 +64,6 @@ namespace librbd { vector >& be, librados::snap_t snap_id, bool sparse, Context *completion, int op_flags); - virtual ~AioObjectRead(); virtual bool should_complete(int r); virtual void send(); @@ -110,6 +110,7 @@ namespace librbd { read_state_d m_state; void send_copyup(); + void read_from_parent(const vector >& image_extents); }; diff --git a/src/librbd/CopyupRequest.cc b/src/librbd/CopyupRequest.cc index 1c1beaa5d70..91e997f2494 100644 --- a/src/librbd/CopyupRequest.cc +++ b/src/librbd/CopyupRequest.cc @@ -177,8 +177,7 @@ private: void CopyupRequest::send() { m_state = STATE_READ_FROM_PARENT; - AioCompletion *comp = aio_create_completion_internal( - util::create_context_callback(this), rbd_ctx_cb); + AioCompletion *comp = AioCompletion::create(this); ldout(m_ictx->cct, 20) << __func__ << " " << this << ": completion " << comp diff --git a/src/librbd/JournalReplay.cc b/src/librbd/JournalReplay.cc index 8a6bff25386..3379f11cf32 100644 --- a/src/librbd/JournalReplay.cc +++ b/src/librbd/JournalReplay.cc @@ -150,8 +150,8 @@ void JournalReplay::handle_event(const journal::UnknownEvent &event, AioCompletion *JournalReplay::create_aio_completion(Context *on_safe) { Mutex::Locker locker(m_lock); - AioCompletion *aio_comp = aio_create_completion_internal( - this, &aio_completion_callback); + AioCompletion *aio_comp = AioCompletion::create(this, aio_completion_callback, + nullptr); m_aio_completions.insert(std::pair( aio_comp, on_safe)); return aio_comp; diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index ca514c8086a..8ee94a5af29 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -2818,7 +2818,7 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) { } Context *ctx = new C_CopyWrite(m_throttle, m_bl); - AioCompletion *comp = aio_create_completion_internal(ctx, rbd_ctx_cb); + AioCompletion *comp = AioCompletion::create(ctx); // coordinate through AIO WQ to ensure lock is acquired if needed m_dest->aio_work_queue->aio_write(comp, m_offset, m_bl->length(), @@ -2876,7 +2876,7 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) { uint64_t len = min(period, src_size - offset); bufferlist *bl = new bufferlist(); Context *ctx = new C_CopyRead(&throttle, dest, offset, bl); - AioCompletion *comp = aio_create_completion_internal(ctx, rbd_ctx_cb); + AioCompletion *comp = AioCompletion::create(ctx); AioImageRequest::aio_read(src, comp, offset, len, NULL, bl, fadvise_flags); prog_ctx.update_progress(offset, src_size); @@ -3440,26 +3440,19 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) { bufferlist bl; - Mutex mylock("IoCtxImpl::write::mylock"); - Cond cond; - bool done; - int ret; - - Context *ctx = new C_SafeCond(&mylock, &cond, &done, &ret); - AioCompletion *c = aio_create_completion_internal(ctx, rbd_ctx_cb); + C_SaferCond ctx; + AioCompletion *c = AioCompletion::create(&ctx); AioImageRequest::aio_read(ictx, c, off, read_len, NULL, &bl, 0); - mylock.Lock(); - while (!done) - cond.Wait(mylock); - mylock.Unlock(); - - if (ret < 0) - return ret; + int ret = ctx.wait(); + if (ret < 0) { + return ret; + } r = cb(total_read, ret, bl.c_str(), arg); - if (r < 0) + if (r < 0) { return r; + } total_read += ret; left -= ret; @@ -3824,22 +3817,4 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) { ictx->perfcounter->inc(l_librbd_readahead_bytes, readahead_length); } } - - AioCompletion *aio_create_completion() { - AioCompletion *c = new AioCompletion(); - return c; - } - - AioCompletion *aio_create_completion(void *cb_arg, callback_t cb_complete) { - AioCompletion *c = new AioCompletion(); - c->set_complete_cb(cb_arg, cb_complete); - return c; - } - - AioCompletion *aio_create_completion_internal(void *cb_arg, - callback_t cb_complete) { - AioCompletion *c = aio_create_completion(cb_arg, cb_complete); - c->rbd_comp = c; - return c; - } } diff --git a/src/librbd/internal.h b/src/librbd/internal.h index b562807e32b..a672b52a3eb 100644 --- a/src/librbd/internal.h +++ b/src/librbd/internal.h @@ -229,14 +229,6 @@ namespace librbd { int mirror_peer_set_cluster(IoCtx& io_ctx, const std::string &cluster_uuid, const std::string &cluster_name); - AioCompletion *aio_create_completion(); - AioCompletion *aio_create_completion(void *cb_arg, callback_t cb_complete); - AioCompletion *aio_create_completion_internal(void *cb_arg, - callback_t cb_complete); - - // raw callbacks - void rbd_req_cb(completion_t cb, void *arg); - void rbd_ctx_cb(completion_t cb, void *arg); } #endif diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index 00a5e477a9f..92b094a0354 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -314,10 +314,8 @@ namespace librbd { RBD::AioCompletion::AioCompletion(void *cb_arg, callback_t complete_cb) { - librbd::AioCompletion *c = librbd::aio_create_completion(cb_arg, - complete_cb); - pc = (void *)c; - c->rbd_comp = this; + pc = reinterpret_cast(librbd::AioCompletion::create( + cb_arg, complete_cb, this)); } bool RBD::AioCompletion::is_complete() -- 2.39.5