From: Mykola Golub Date: Sun, 17 Jan 2016 11:24:36 +0000 (+0200) Subject: librbd: API: async open and close X-Git-Tag: v10.0.4~149^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=458d8e59cfc418cfbeacd9a3de57ecd98ed1ff6e;p=ceph.git librbd: API: async open and close Fixes: #14264 Signed-off-by: Mykola Golub --- diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index e2d63f9a095f..dde1a61934bd 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -41,6 +41,7 @@ extern "C" { #define LIBRBD_SUPPORTS_WATCH 0 #define LIBRBD_SUPPORTS_AIO_FLUSH 1 #define LIBRBD_SUPPORTS_INVALIDATE 1 +#define LIBRBD_SUPPORTS_AIO_OPEN 1 #if __GNUC__ >= 4 #define CEPH_RBD_API __attribute__ ((visibility ("default"))) @@ -55,6 +56,9 @@ typedef void *rbd_snap_t; typedef void *rbd_image_t; typedef void *rbd_image_options_t; +typedef void *rbd_completion_t; +typedef void (*rbd_callback_t)(rbd_completion_t cb, void *arg); + typedef int (*librbd_progress_fn_t)(uint64_t offset, uint64_t total, void *ptr); typedef struct { @@ -188,6 +192,9 @@ CEPH_RBD_API int rbd_mirror_peer_set_cluster(rados_ioctx_t io_ctx, CEPH_RBD_API int rbd_open(rados_ioctx_t io, const char *name, rbd_image_t *image, const char *snap_name); +CEPH_RBD_API int rbd_aio_open(rados_ioctx_t io, const char *name, + rbd_image_t *image, const char *snap_name, + rbd_completion_t c); /** * Open an image in read-only mode. * @@ -209,7 +216,11 @@ CEPH_RBD_API int rbd_open(rados_ioctx_t io, const char *name, */ CEPH_RBD_API int rbd_open_read_only(rados_ioctx_t io, const char *name, rbd_image_t *image, const char *snap_name); +CEPH_RBD_API int rbd_aio_open_read_only(rados_ioctx_t io, const char *name, + rbd_image_t *image, const char *snap_name, + rbd_completion_t c); CEPH_RBD_API int rbd_close(rbd_image_t image); +CEPH_RBD_API int rbd_aio_close(rbd_image_t image, rbd_completion_t c); CEPH_RBD_API int rbd_resize(rbd_image_t image, uint64_t size); CEPH_RBD_API int rbd_resize_with_progress(rbd_image_t image, uint64_t size, librbd_progress_fn_t cb, void *cbdata); @@ -423,8 +434,6 @@ CEPH_RBD_API int rbd_break_lock(rbd_image_t image, const char *client, /** @} locking */ /* I/O */ -typedef void *rbd_completion_t; -typedef void (*rbd_callback_t)(rbd_completion_t cb, void *arg); CEPH_RBD_API ssize_t rbd_read(rbd_image_t image, uint64_t ofs, size_t len, char *buf); /* diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp index d9bf1de65d09..7a4a2c7bdda6 100644 --- a/src/include/rbd/librbd.hpp +++ b/src/include/rbd/librbd.hpp @@ -84,9 +84,13 @@ public: int open(IoCtx& io_ctx, Image& image, const char *name); int open(IoCtx& io_ctx, Image& image, const char *name, const char *snapname); + int aio_open(IoCtx& io_ctx, Image& image, const char *name, + const char *snapname, RBD::AioCompletion *c); // see librbd.h int open_read_only(IoCtx& io_ctx, Image& image, const char *name, const char *snapname); + int aio_open_read_only(IoCtx& io_ctx, Image& image, const char *name, + const char *snapname, RBD::AioCompletion *c); int list(IoCtx& io_ctx, std::vector& names); int create(IoCtx& io_ctx, const char *name, uint64_t size, int *order); int create2(IoCtx& io_ctx, const char *name, uint64_t size, @@ -155,6 +159,7 @@ public: ~Image(); int close(); + int aio_close(RBD::AioCompletion *c); int resize(uint64_t size); int resize_with_progress(uint64_t size, ProgressContext& pctx); diff --git a/src/librbd/AioCompletion.cc b/src/librbd/AioCompletion.cc index 9ad318bc51ec..ab71c3c268ea 100644 --- a/src/librbd/AioCompletion.cc +++ b/src/librbd/AioCompletion.cc @@ -69,6 +69,9 @@ namespace librbd { assert(lock.is_locked()); elapsed = ceph_clock_now(cct) - start_time; switch (aio_type) { + case AIO_TYPE_OPEN: + case AIO_TYPE_CLOSE: + break; case AIO_TYPE_READ: ictx->perfcounter->tinc(l_librbd_rd_latency, elapsed); break; case AIO_TYPE_WRITE: diff --git a/src/librbd/AioCompletion.h b/src/librbd/AioCompletion.h index 0d1e226061ba..c554bc5072ef 100644 --- a/src/librbd/AioCompletion.h +++ b/src/librbd/AioCompletion.h @@ -21,11 +21,13 @@ namespace librbd { class AioObjectRead; typedef enum { - AIO_TYPE_READ = 0, + AIO_TYPE_NONE = 0, + AIO_TYPE_OPEN, + AIO_TYPE_CLOSE, + AIO_TYPE_READ, AIO_TYPE_WRITE, AIO_TYPE_DISCARD, AIO_TYPE_FLUSH, - AIO_TYPE_NONE, } aio_type_t; /** @@ -156,11 +158,17 @@ namespace librbd { int n = --ref; lock.Unlock(); if (!n) { - if (ictx && event_notify) { - ictx->completed_reqs_lock.Lock(); - m_xlist_item.remove_myself(); - ictx->completed_reqs_lock.Unlock(); - } + if (ictx) { + if (event_notify) { + ictx->completed_reqs_lock.Lock(); + m_xlist_item.remove_myself(); + ictx->completed_reqs_lock.Unlock(); + } + if (aio_type == AIO_TYPE_CLOSE || (aio_type == AIO_TYPE_OPEN && + rval < 0)) { + delete ictx; + } + } delete this; } } diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index 3fda9db3ce5d..0b76ebae0524 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -69,6 +69,69 @@ librbd::AioCompletion* get_aio_completion(librbd::RBD::AioCompletion *comp) { return reinterpret_cast(comp->pc); } +struct C_OpenComplete : public Context { + librbd::ImageCtx *ictx; + librbd::AioCompletion* comp; + void **ictxp; + bool reopen; + C_OpenComplete(librbd::ImageCtx *ictx, librbd::AioCompletion* comp, + void **ictxp, bool reopen = false) + : ictx(ictx), comp(comp), ictxp(ictxp), reopen(reopen) { + comp->init_time(ictx, librbd::AIO_TYPE_OPEN); + comp->get(); + } + virtual void finish(int r) { + ldout(ictx->cct, 20) << "C_OpenComplete::finish: r=" << r << dendl; + if (reopen) { + delete reinterpret_cast(*ictxp); + } + if (r < 0) { + *ictxp = nullptr; + comp->fail(ictx->cct, r); + } else { + *ictxp = ictx; + comp->lock.Lock(); + comp->complete(ictx->cct); + comp->put_unlock(); + } + } +}; + +struct C_OpenAfterCloseComplete : public Context { + librbd::ImageCtx *ictx; + librbd::AioCompletion* comp; + void **ictxp; + C_OpenAfterCloseComplete(librbd::ImageCtx *ictx, librbd::AioCompletion* comp, + void **ictxp) + : ictx(ictx), comp(comp), ictxp(ictxp) { + } + virtual void finish(int r) { + ldout(ictx->cct, 20) << "C_OpenAfterCloseComplete::finish: r=" << r + << dendl; + ictx->state->open(new C_OpenComplete(ictx, comp, ictxp, true)); + } +}; + +struct C_CloseComplete : public Context { + CephContext *cct; + librbd::AioCompletion* comp; + C_CloseComplete(librbd::ImageCtx *ictx, librbd::AioCompletion* comp) + : cct(ictx->cct), comp(comp) { + comp->init_time(ictx, librbd::AIO_TYPE_CLOSE); + comp->get(); + } + virtual void finish(int r) { + ldout(cct, 20) << "C_CloseComplete::finish: r=" << r << dendl; + if (r < 0) { + comp->fail(cct, r); + } else { + comp->lock.Lock(); + comp->complete(cct); + comp->put_unlock(); + } + } +}; + } // anonymous namespace namespace librbd { @@ -137,6 +200,24 @@ namespace librbd { return 0; } + int RBD::aio_open(IoCtx& io_ctx, Image& image, const char *name, + const char *snap_name, RBD::AioCompletion *c) + { + ImageCtx *ictx = new ImageCtx(name, "", snap_name, io_ctx, false); + TracepointProvider::initialize(get_cct(io_ctx)); + tracepoint(librbd, aio_open_image_enter, ictx, ictx->name.c_str(), ictx->id.c_str(), ictx->snap_name.c_str(), ictx->read_only, c->pc); + + if (image.ctx != NULL) { + reinterpret_cast(image.ctx)->state->close( + new C_OpenAfterCloseComplete(ictx, get_aio_completion(c), &image.ctx)); + } else { + ictx->state->open(new C_OpenComplete(ictx, get_aio_completion(c), + &image.ctx)); + } + tracepoint(librbd, aio_open_image_exit, 0); + return 0; + } + int RBD::open_read_only(IoCtx& io_ctx, Image& image, const char *name, const char *snap_name) { @@ -161,6 +242,24 @@ namespace librbd { return 0; } + int RBD::aio_open_read_only(IoCtx& io_ctx, Image& image, const char *name, + const char *snap_name, RBD::AioCompletion *c) + { + ImageCtx *ictx = new ImageCtx(name, "", snap_name, io_ctx, true); + TracepointProvider::initialize(get_cct(io_ctx)); + tracepoint(librbd, aio_open_image_enter, ictx, ictx->name.c_str(), ictx->id.c_str(), ictx->snap_name.c_str(), ictx->read_only, c->pc); + + if (image.ctx != NULL) { + reinterpret_cast(image.ctx)->state->close( + new C_OpenAfterCloseComplete(ictx, get_aio_completion(c), &image.ctx)); + } else { + ictx->state->open(new C_OpenComplete(ictx, get_aio_completion(c), + &image.ctx)); + } + tracepoint(librbd, aio_open_image_exit, 0); + return 0; + } + int RBD::create(IoCtx& io_ctx, const char *name, uint64_t size, int *order) { TracepointProvider::initialize(get_cct(io_ctx)); @@ -434,6 +533,22 @@ namespace librbd { return r; } + int Image::aio_close(RBD::AioCompletion *c) + { + if (!ctx) { + return -EINVAL; + } + + ImageCtx *ictx = (ImageCtx *)ctx; + tracepoint(librbd, aio_close_image_enter, ictx, ictx->name.c_str(), ictx->id.c_str(), c->pc); + + ictx->state->close(new C_CloseComplete(ictx, get_aio_completion(c))); + ctx = NULL; + + tracepoint(librbd, aio_close_image_exit, 0); + return 0; + } + int Image::resize(uint64_t size) { ImageCtx *ictx = (ImageCtx *)ctx; @@ -1575,6 +1690,22 @@ extern "C" int rbd_open(rados_ioctx_t p, const char *name, rbd_image_t *image, return r; } +extern "C" int rbd_aio_open(rados_ioctx_t p, const char *name, + rbd_image_t *image, const char *snap_name, + rbd_completion_t c) +{ + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + TracepointProvider::initialize(get_cct(io_ctx)); + librbd::ImageCtx *ictx = new librbd::ImageCtx(name, "", snap_name, io_ctx, + false); + librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c; + tracepoint(librbd, aio_open_image_enter, ictx, ictx->name.c_str(), ictx->id.c_str(), ictx->snap_name.c_str(), ictx->read_only, comp->pc); + ictx->state->open(new C_OpenComplete(ictx, get_aio_completion(comp), image)); + tracepoint(librbd, aio_open_image_exit, 0); + return 0; +} + extern "C" int rbd_open_read_only(rados_ioctx_t p, const char *name, rbd_image_t *image, const char *snap_name) { @@ -1595,6 +1726,22 @@ extern "C" int rbd_open_read_only(rados_ioctx_t p, const char *name, return r; } +extern "C" int rbd_aio_open_read_only(rados_ioctx_t p, const char *name, + rbd_image_t *image, const char *snap_name, + rbd_completion_t c) +{ + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + TracepointProvider::initialize(get_cct(io_ctx)); + librbd::ImageCtx *ictx = new librbd::ImageCtx(name, "", snap_name, io_ctx, + true); + librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c; + tracepoint(librbd, aio_open_image_enter, ictx, ictx->name.c_str(), ictx->id.c_str(), ictx->snap_name.c_str(), ictx->read_only, comp->pc); + ictx->state->open(new C_OpenComplete(ictx, get_aio_completion(comp), image)); + tracepoint(librbd, aio_open_image_exit, 0); + return 0; +} + extern "C" int rbd_close(rbd_image_t image) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; @@ -1606,6 +1753,16 @@ extern "C" int rbd_close(rbd_image_t image) return r; } +extern "C" int rbd_aio_close(rbd_image_t image, rbd_completion_t c) +{ + librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; + librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c; + tracepoint(librbd, aio_close_image_enter, ictx, ictx->name.c_str(), ictx->id.c_str(), comp->pc); + ictx->state->close(new C_CloseComplete(ictx, get_aio_completion(comp))); + tracepoint(librbd, aio_close_image_exit, 0); + return 0; +} + extern "C" int rbd_resize(rbd_image_t image, uint64_t size) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc index 15f5084f9e12..e8b21d6ee5c1 100644 --- a/src/test/librbd/test_librbd.cc +++ b/src/test/librbd/test_librbd.cc @@ -285,6 +285,134 @@ TEST_F(TestLibRBD, CreateAndStatPP) ioctx.close(); } +TEST_F(TestLibRBD, OpenAio) +{ + rados_ioctx_t ioctx; + ASSERT_EQ(0, rados_ioctx_create(_cluster, m_pool_name.c_str(), &ioctx)); + + rbd_image_info_t info; + rbd_image_t image; + int order = 0; + std::string name = get_temp_image_name(); + uint64_t size = 2 << 20; + + ASSERT_EQ(0, create_image(ioctx, name.c_str(), size, &order)); + + rbd_completion_t open_comp; + ASSERT_EQ(0, rbd_aio_create_completion(NULL, NULL, &open_comp)); + ASSERT_EQ(0, rbd_aio_open(ioctx, name.c_str(), &image, NULL, open_comp)); + ASSERT_EQ(0, rbd_aio_wait_for_complete(open_comp)); + ASSERT_EQ(1, rbd_aio_is_complete(open_comp)); + ASSERT_EQ(0, rbd_aio_get_return_value(open_comp)); + rbd_aio_release(open_comp); + + ASSERT_EQ(0, rbd_stat(image, &info, sizeof(info))); + printf("image has size %llu and order %d\n", (unsigned long long) info.size, info.order); + ASSERT_EQ(info.size, size); + ASSERT_EQ(info.order, order); + + rbd_completion_t close_comp; + ASSERT_EQ(0, rbd_aio_create_completion(NULL, NULL, &close_comp)); + ASSERT_EQ(0, rbd_aio_close(image, close_comp)); + ASSERT_EQ(0, rbd_aio_wait_for_complete(close_comp)); + ASSERT_EQ(1, rbd_aio_is_complete(close_comp)); + ASSERT_EQ(0, rbd_aio_get_return_value(close_comp)); + rbd_aio_release(close_comp); + + rados_ioctx_destroy(ioctx); +} + +TEST_F(TestLibRBD, OpenAioFail) +{ + rados_ioctx_t ioctx; + ASSERT_EQ(0, rados_ioctx_create(_cluster, m_pool_name.c_str(), &ioctx)); + + std::string name = get_temp_image_name(); + rbd_image_t image; + rbd_completion_t open_comp; + ASSERT_EQ(0, rbd_aio_create_completion(NULL, NULL, &open_comp)); + ASSERT_EQ(0, rbd_aio_open(ioctx, name.c_str(), &image, NULL, open_comp)); + ASSERT_EQ(0, rbd_aio_wait_for_complete(open_comp)); + ASSERT_EQ(1, rbd_aio_is_complete(open_comp)); + ASSERT_EQ(-ENOENT, rbd_aio_get_return_value(open_comp)); + rbd_aio_release(open_comp); + + rados_ioctx_destroy(ioctx); +} + +TEST_F(TestLibRBD, OpenAioPP) +{ + librados::IoCtx ioctx; + ASSERT_EQ(0, _rados.ioctx_create(m_pool_name.c_str(), ioctx)); + + librbd::RBD rbd; + librbd::image_info_t info; + librbd::Image image; + int order = 0; + std::string name = get_temp_image_name(); + uint64_t size = 2 << 20; + + ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order)); + + librbd::RBD::AioCompletion *open_comp = + new librbd::RBD::AioCompletion(NULL, NULL); + ASSERT_EQ(0, rbd.aio_open(ioctx, image, name.c_str(), NULL, open_comp)); + ASSERT_EQ(0, open_comp->wait_for_complete()); + ASSERT_EQ(1, open_comp->is_complete()); + ASSERT_EQ(0, open_comp->get_return_value()); + open_comp->release(); + + ASSERT_EQ(0, image.stat(info, sizeof(info))); + ASSERT_EQ(info.size, size); + ASSERT_EQ(info.order, order); + + // reopen + open_comp = new librbd::RBD::AioCompletion(NULL, NULL); + ASSERT_EQ(0, rbd.aio_open(ioctx, image, name.c_str(), NULL, open_comp)); + ASSERT_EQ(0, open_comp->wait_for_complete()); + ASSERT_EQ(1, open_comp->is_complete()); + ASSERT_EQ(0, open_comp->get_return_value()); + open_comp->release(); + + // close + librbd::RBD::AioCompletion *close_comp = + new librbd::RBD::AioCompletion(NULL, NULL); + ASSERT_EQ(0, image.aio_close(close_comp)); + ASSERT_EQ(0, close_comp->wait_for_complete()); + ASSERT_EQ(1, close_comp->is_complete()); + ASSERT_EQ(0, close_comp->get_return_value()); + close_comp->release(); + + // close closed image + close_comp = new librbd::RBD::AioCompletion(NULL, NULL); + ASSERT_EQ(-EINVAL, image.aio_close(close_comp)); + close_comp->release(); + + ioctx.close(); +} + +TEST_F(TestLibRBD, OpenAioFailPP) +{ + librados::IoCtx ioctx; + ASSERT_EQ(0, _rados.ioctx_create(m_pool_name.c_str(), ioctx)); + + { + librbd::RBD rbd; + librbd::Image image; + std::string name = get_temp_image_name(); + + librbd::RBD::AioCompletion *open_comp = + new librbd::RBD::AioCompletion(NULL, NULL); + ASSERT_EQ(0, rbd.aio_open(ioctx, image, name.c_str(), NULL, open_comp)); + ASSERT_EQ(0, open_comp->wait_for_complete()); + ASSERT_EQ(1, open_comp->is_complete()); + ASSERT_EQ(-ENOENT, open_comp->get_return_value()); + open_comp->release(); + } + + ioctx.close(); +} + TEST_F(TestLibRBD, ResizeAndStat) { rados_ioctx_t ioctx; diff --git a/src/tracing/librbd.tp b/src/tracing/librbd.tp index 58cbc13a78d8..f91e4e3ee93b 100644 --- a/src/tracing/librbd.tp +++ b/src/tracing/librbd.tp @@ -172,6 +172,32 @@ TRACEPOINT_EVENT(librbd, open_image_exit, ) ) +TRACEPOINT_EVENT(librbd, aio_open_image_enter, + TP_ARGS( + void*, imagectx, + const char*, name, + const char*, id, + const char*, snap_name, + int, read_only, + const void*, completion), + TP_FIELDS( + ctf_integer_hex(void*, imagectx, imagectx) + ctf_string(name, name) + ctf_string(id, id) + ctf_string(snap_name, snap_name) + ctf_integer(uint8_t, read_only, read_only ? 1 : 0) + ctf_integer_hex(const void*, completion, completion) + ) +) + +TRACEPOINT_EVENT(librbd, aio_open_image_exit, + TP_ARGS( + int, retval), + TP_FIELDS( + ctf_integer(int, retval, retval) + ) +) + TRACEPOINT_EVENT(librbd, close_image_enter, TP_ARGS( void*, imagectx, @@ -191,6 +217,27 @@ TRACEPOINT_EVENT(librbd, close_image_exit, ctf_integer(int, retval, retval)) ) +TRACEPOINT_EVENT(librbd, aio_close_image_enter, + TP_ARGS( + void*, imagectx, + const char*, name, + const char*, id, + const void*, completion), + TP_FIELDS( + ctf_integer_hex(void*, imagectx, imagectx) + ctf_string(name, name) + ctf_string(id, id) + ctf_integer_hex(const void*, completion, completion) + ) +) + +TRACEPOINT_EVENT(librbd, aio_close_image_exit, + TP_ARGS( + int, retval), + TP_FIELDS( + ctf_integer(int, retval, retval)) +) + TRACEPOINT_EVENT(librbd, list_enter, TP_ARGS( const char*, pool_name,