From 4501e90d9b667310a222c3616a7b1f1fb382f997 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Mon, 1 Jun 2015 22:56:11 -0400 Subject: [PATCH] librbd: new rbd_non_blocking_aio config option Setting this option to false reverts librbd to legacy behavior where AIO operations could potentially block. Signed-off-by: Jason Dillaman (cherry picked from commit 769cad12716b85d87eacc1069dd9f5c21cad3915) Conflicts: src/common/config_opts.h: trivial resolution src/librbd/librbd.cc: trivial resolution --- src/common/config_opts.h | 1 + src/librbd/librbd.cc | 57 ++++++++++++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 2290c569fbf1a..23fbb9437b220 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -729,6 +729,7 @@ OPTION(rados_osd_op_timeout, OPT_DOUBLE, 0) // how many seconds to wait for a re OPTION(rbd_op_threads, OPT_INT, 1) OPTION(rbd_op_thread_timeout, OPT_INT, 60) +OPTION(rbd_non_blocking_aio, OPT_BOOL, true) // process AIO ops from a worker thread to prevent blocking OPTION(rbd_cache, OPT_BOOL, false) // whether to enable caching (writeback unless rbd_cache_max_dirty is 0) OPTION(rbd_cache_writethrough_until_flush, OPT_BOOL, false) // whether to make writeback caching writethrough until flush is called, to be sure the user of librbd will send flushs so that writeback is safe OPTION(rbd_cache_size, OPT_LONGLONG, 32<<20) // cache size in bytes diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index 7f966be47d9a5..244a5a0526f6c 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -116,6 +116,41 @@ private: librbd::AioCompletion *m_comp; }; +void submit_aio_read(librbd::ImageCtx *ictx, uint64_t off, size_t len, + char *buf, bufferlist *pbl, librbd::AioCompletion *c) { + if (ictx->cct->_conf->rbd_non_blocking_aio) { + ictx->aio_work_queue->queue(new C_AioReadWQ(ictx, off, len, buf, pbl, c)); + } else { + librbd::aio_read(ictx, off, len, buf, pbl, c); + } +} + +void submit_aio_write(librbd::ImageCtx *ictx, uint64_t off, size_t len, + const char *buf, librbd::AioCompletion *c) { + if (ictx->cct->_conf->rbd_non_blocking_aio) { + ictx->aio_work_queue->queue(new C_AioWriteWQ(ictx, off, len, buf, c)); + } else { + librbd::aio_write(ictx, off, len, buf, c); + } +} + +void submit_aio_discard(librbd::ImageCtx *ictx, uint64_t off, uint64_t len, + librbd::AioCompletion *c) { + if (ictx->cct->_conf->rbd_non_blocking_aio) { + ictx->aio_work_queue->queue(new C_AioDiscardWQ(ictx, off, len, c)); + } else { + librbd::aio_discard(ictx, off, len, c); + } +} + +void submit_aio_flush(librbd::ImageCtx *ictx, librbd::AioCompletion *c) { + if (ictx->cct->_conf->rbd_non_blocking_aio) { + ictx->aio_work_queue->queue(new C_AioFlushWQ(ictx, c)); + } else { + librbd::aio_flush(ictx, c); + } +} + librbd::AioCompletion* get_aio_completion(librbd::RBD::AioCompletion *comp) { return reinterpret_cast(comp->pc); } @@ -560,16 +595,14 @@ namespace librbd { ImageCtx *ictx = (ImageCtx *)ctx; if (bl.length() < len) return -EINVAL; - ictx->aio_work_queue->queue(new C_AioWriteWQ(ictx, off, len, bl.c_str(), - get_aio_completion(c))); + submit_aio_write(ictx, off, len, bl.c_str(), get_aio_completion(c)); return 0; } int Image::aio_discard(uint64_t off, uint64_t len, RBD::AioCompletion *c) { ImageCtx *ictx = (ImageCtx *)ctx; - ictx->aio_work_queue->queue(new C_AioDiscardWQ(ictx, off, len, - get_aio_completion(c))); + submit_aio_discard(ictx, off, len, get_aio_completion(c)); return 0; } @@ -579,8 +612,7 @@ namespace librbd { ImageCtx *ictx = (ImageCtx *)ctx; ldout(ictx->cct, 10) << "Image::aio_read() buf=" << (void *)bl.c_str() << "~" << (void *)(bl.c_str() + len - 1) << dendl; - ictx->aio_work_queue->queue(new C_AioReadWQ(ictx, off, len, NULL, &bl, - get_aio_completion(c))); + submit_aio_read(ictx, off, len, NULL, &bl, get_aio_completion(c)); return 0; } @@ -593,7 +625,7 @@ namespace librbd { int Image::aio_flush(RBD::AioCompletion *c) { ImageCtx *ictx = (ImageCtx *)ctx; - ictx->aio_work_queue->queue(new C_AioFlushWQ(ictx, get_aio_completion(c))); + submit_aio_flush(ictx, get_aio_completion(c)); return 0; } @@ -1185,8 +1217,7 @@ extern "C" int rbd_aio_write(rbd_image_t image, uint64_t off, size_t len, { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c; - ictx->aio_work_queue->queue(new C_AioWriteWQ(ictx, off, len, buf, - get_aio_completion(comp))); + submit_aio_write(ictx, off, len, buf, get_aio_completion(comp)); return 0; } @@ -1195,8 +1226,7 @@ extern "C" int rbd_aio_discard(rbd_image_t image, uint64_t off, uint64_t len, { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c; - ictx->aio_work_queue->queue(new C_AioDiscardWQ(ictx, off, len, - get_aio_completion(comp))); + submit_aio_discard(ictx, off, len, get_aio_completion(comp)); return 0; } @@ -1205,8 +1235,7 @@ extern "C" int rbd_aio_read(rbd_image_t image, uint64_t off, size_t len, { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c; - ictx->aio_work_queue->queue(new C_AioReadWQ(ictx, off, len, buf, NULL, - get_aio_completion(comp))); + submit_aio_read(ictx, off, len, buf, NULL, get_aio_completion(comp)); return 0; } @@ -1220,7 +1249,7 @@ extern "C" int rbd_aio_flush(rbd_image_t image, rbd_completion_t c) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c; - ictx->aio_work_queue->queue(new C_AioFlushWQ(ictx, get_aio_completion(comp))); + submit_aio_flush(ictx, get_aio_completion(comp)); return 0; } -- 2.39.5