librbd: new rbd_non_blocking_aio config option

Setting this option to false reverts librbd to legacy behavior
where AIO operations could potentially block.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit 769cad12716b85d87eacc1069dd9f5c21cad3915)

Conflicts:
	src/common/config_opts.h: trivial resolution
	src/librbd/librbd.cc: trivial resolution

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 2290c569fbf1a291f70523232f7f24ff12e3da4a..23fbb9437b220772ae5d5bb1914a2f969d23f1dc 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -729,6 +729,7 @@ OPTION(rados_osd_op_timeout, OPT_DOUBLE, 0) // how many seconds to wait for a re
 
 OPTION(rbd_op_threads, OPT_INT, 1)
 OPTION(rbd_op_thread_timeout, OPT_INT, 60)
+OPTION(rbd_non_blocking_aio, OPT_BOOL, true) // process AIO ops from a worker thread to prevent blocking
 OPTION(rbd_cache, OPT_BOOL, false) // whether to enable caching (writeback unless rbd_cache_max_dirty is 0)
 OPTION(rbd_cache_writethrough_until_flush, OPT_BOOL, false) // whether to make writeback caching writethrough until flush is called, to be sure the user of librbd will send flushs so that writeback is safe
 OPTION(rbd_cache_size, OPT_LONGLONG, 32<<20)         // cache size in bytes

diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc
index 7f966be47d9a53b074d086d51a4dcdecdee3ca5b..244a5a0526f6c4e1d3d43b57b51e8ca83d910aa4 100644 (file)
--- a/src/librbd/librbd.cc
+++ b/src/librbd/librbd.cc
@@ -116,6 +116,41 @@ private:
   librbd::AioCompletion *m_comp;
 };
 
+void submit_aio_read(librbd::ImageCtx *ictx, uint64_t off, size_t len,
+                     char *buf, bufferlist *pbl, librbd::AioCompletion *c) {
+  if (ictx->cct->_conf->rbd_non_blocking_aio) {
+    ictx->aio_work_queue->queue(new C_AioReadWQ(ictx, off, len, buf, pbl, c));
+  } else {
+    librbd::aio_read(ictx, off, len, buf, pbl, c);
+  }
+}
+
+void submit_aio_write(librbd::ImageCtx *ictx, uint64_t off, size_t len,
+                      const char *buf, librbd::AioCompletion *c) {
+  if (ictx->cct->_conf->rbd_non_blocking_aio) {
+    ictx->aio_work_queue->queue(new C_AioWriteWQ(ictx, off, len, buf, c));
+  } else {
+    librbd::aio_write(ictx, off, len, buf, c);
+  }
+}
+
+void submit_aio_discard(librbd::ImageCtx *ictx, uint64_t off, uint64_t len,
+                        librbd::AioCompletion *c) {
+  if (ictx->cct->_conf->rbd_non_blocking_aio) {
+    ictx->aio_work_queue->queue(new C_AioDiscardWQ(ictx, off, len, c));
+  } else {
+    librbd::aio_discard(ictx, off, len, c);
+  }
+}
+
+void submit_aio_flush(librbd::ImageCtx *ictx, librbd::AioCompletion *c) {
+  if (ictx->cct->_conf->rbd_non_blocking_aio) {
+    ictx->aio_work_queue->queue(new C_AioFlushWQ(ictx, c));
+  } else {
+    librbd::aio_flush(ictx, c);
+  }
+}
+
 librbd::AioCompletion* get_aio_completion(librbd::RBD::AioCompletion *comp) {
   return reinterpret_cast<librbd::AioCompletion *>(comp->pc);
 }
@@ -560,16 +595,14 @@ namespace librbd {
     ImageCtx *ictx = (ImageCtx *)ctx;
     if (bl.length() < len)
       return -EINVAL;
-    ictx->aio_work_queue->queue(new C_AioWriteWQ(ictx, off, len, bl.c_str(),
-                                                 get_aio_completion(c)));
+    submit_aio_write(ictx, off, len, bl.c_str(), get_aio_completion(c));
     return 0;
   }
 
   int Image::aio_discard(uint64_t off, uint64_t len, RBD::AioCompletion *c)
   {
     ImageCtx *ictx = (ImageCtx *)ctx;
-    ictx->aio_work_queue->queue(new C_AioDiscardWQ(ictx, off, len,
-                                                   get_aio_completion(c)));
+    submit_aio_discard(ictx, off, len, get_aio_completion(c));
     return 0;
   }
 
@@ -579,8 +612,7 @@ namespace librbd {
     ImageCtx *ictx = (ImageCtx *)ctx;
     ldout(ictx->cct, 10) << "Image::aio_read() buf=" << (void *)bl.c_str() << "~"
                         << (void *)(bl.c_str() + len - 1) << dendl;
-    ictx->aio_work_queue->queue(new C_AioReadWQ(ictx, off, len, NULL, &bl,
-                                                get_aio_completion(c)));
+    submit_aio_read(ictx, off, len, NULL, &bl, get_aio_completion(c));
     return 0;
   }
 
@@ -593,7 +625,7 @@ namespace librbd {
   int Image::aio_flush(RBD::AioCompletion *c)
   {
     ImageCtx *ictx = (ImageCtx *)ctx;
-    ictx->aio_work_queue->queue(new C_AioFlushWQ(ictx, get_aio_completion(c)));
+    submit_aio_flush(ictx, get_aio_completion(c));
     return 0;
   }
 
@@ -1185,8 +1217,7 @@ extern "C" int rbd_aio_write(rbd_image_t image, uint64_t off, size_t len,
 {
   librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
   librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;
-  ictx->aio_work_queue->queue(new C_AioWriteWQ(ictx, off, len, buf,
-                                               get_aio_completion(comp)));
+  submit_aio_write(ictx, off, len, buf, get_aio_completion(comp));
   return 0;
 }
 
@@ -1195,8 +1226,7 @@ extern "C" int rbd_aio_discard(rbd_image_t image, uint64_t off, uint64_t len,
 {
   librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
   librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;
-  ictx->aio_work_queue->queue(new C_AioDiscardWQ(ictx, off, len,
-                                                 get_aio_completion(comp)));
+  submit_aio_discard(ictx, off, len, get_aio_completion(comp));
   return 0;
 }
 
@@ -1205,8 +1235,7 @@ extern "C" int rbd_aio_read(rbd_image_t image, uint64_t off, size_t len,
 {
   librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
   librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;
-  ictx->aio_work_queue->queue(new C_AioReadWQ(ictx, off, len, buf, NULL,
-                                              get_aio_completion(comp)));
+  submit_aio_read(ictx, off, len, buf, NULL, get_aio_completion(comp));
   return 0;
 }
 
@@ -1220,7 +1249,7 @@ extern "C" int rbd_aio_flush(rbd_image_t image, rbd_completion_t c)
 {
   librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
   librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;
-  ictx->aio_work_queue->queue(new C_AioFlushWQ(ictx, get_aio_completion(comp)));
+  submit_aio_flush(ictx, get_aio_completion(comp));
   return 0;
 }