librbd: delete more than one object at once

Speed up deletions when resizing down or removing an image by keeping
up 10 operations in flight by default.

Refs: #2256
Signed-off-by: Josh Durgin <josh.durgin@inktank.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 84df2bff266643e1dbef01a532cf55060f435941..0a3aa85261e9dffcf568d8cc63973ce8ad8469d4 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -534,6 +534,7 @@ OPTION(rbd_cache_max_dirty, OPT_LONGLONG, 24<<20)    // dirty limit in bytes - s
 OPTION(rbd_cache_target_dirty, OPT_LONGLONG, 16<<20) // target dirty limit in bytes
 OPTION(rbd_cache_max_dirty_age, OPT_FLOAT, 1.0)      // seconds in cache before writeback starts
 OPTION(rbd_cache_block_writes_upfront, OPT_BOOL, false) // whether to block writes to the cache before the aio_write call completes (true), or block before the aio completion is called (false)
+OPTION(rbd_concurrent_management_ops, OPT_INT, 10) // how many operations can be in flight for a management operation like deleting or resizing an image
 
 OPTION(nss_db_path, OPT_STR, "") // path to nss db
 

diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc
index 8873ac69a40306613a87a0f31c301ed7d9599bf9..4d3b55ee070717f33107e473f253d0efab73dc42 100644 (file)
--- a/src/librbd/internal.cc
+++ b/src/librbd/internal.cc
@@ -6,6 +6,7 @@
 #include "common/ceph_context.h"
 #include "common/dout.h"
 #include "common/errno.h"
+#include "common/Throttle.h"
 #include "cls/lock/cls_lock_client.h"
 #include "include/inttypes.h"
 #include "include/stringify.h"
@@ -151,22 +152,30 @@ namespace librbd {
     uint64_t period = ictx->get_stripe_period();
     uint64_t num_period = ((newsize + period - 1) / period);
     uint64_t delete_off = MIN(num_period * period, size);
-    uint64_t delete_start = num_period * ictx->get_stripe_count();     // first object we can delete free and clear
+    // first object we can delete free and clear
+    uint64_t delete_start = num_period * ictx->get_stripe_count();
     uint64_t num_objects = ictx->get_num_objects();
     uint64_t object_size = ictx->get_object_size();
 
     ldout(cct, 10) << "trim_image " << size << " -> " << newsize
                   << " periods " << num_period
                   << " discard to offset " << delete_off
-                  << " delete objects " << delete_start << " to " << (num_objects-1)
+                  << " delete objects " << delete_start
+                  << " to " << (num_objects-1)
                   << dendl;
 
+    SimpleThrottle throttle(cct->_conf->rbd_concurrent_management_ops, true);
     if (delete_start < num_objects) {
       ldout(cct, 2) << "trim_image objects " << delete_start << " to "
                    << (num_objects - 1) << dendl;
       for (uint64_t i = delete_start; i < num_objects; ++i) {
+       throttle.start_op();
        string oid = ictx->get_object_name(i);
-       ictx->data_ctx.remove(oid);
+       Context *req_comp = new C_SimpleThrottle(&throttle);
+       librados::AioCompletion *rados_completion =
+         librados::Rados::aio_create_completion(req_comp, NULL, rados_ctx_cb);
+       ictx->data_ctx.aio_remove(oid, rados_completion);
+       rados_completion->release();
        prog_ctx.update_progress((i - delete_start) * object_size,
                                 (num_objects - delete_start) * object_size);
       }
@@ -175,19 +184,31 @@ namespace librbd {
     // discard the weird boundary, if any
     if (delete_off > newsize) {
       vector<ObjectExtent> extents;
-      Striper::file_to_extents(ictx->cct, ictx->format_string, &ictx->layout, newsize, delete_off - newsize, extents);
+      Striper::file_to_extents(ictx->cct, ictx->format_string, &ictx->layout,
+                              newsize, delete_off - newsize, extents);
 
-      for (vector<ObjectExtent>::iterator p = extents.begin(); p != extents.end(); ++p) {
+      for (vector<ObjectExtent>::iterator p = extents.begin();
+          p != extents.end(); ++p) {
        ldout(ictx->cct, 20) << " ex " << *p << dendl;
+       throttle.start_op();
+       Context *req_comp = new C_SimpleThrottle(&throttle);
+       librados::AioCompletion *rados_completion =
+         librados::Rados::aio_create_completion(req_comp, NULL, rados_ctx_cb);
        if (p->offset == 0) {
-         ictx->data_ctx.remove(p->oid.name);
+         ictx->data_ctx.aio_remove(p->oid.name, rados_completion);
        } else {
          librados::ObjectWriteOperation op;
          op.truncate(p->offset);
-         ictx->data_ctx.operate(p->oid.name, &op);
+         ictx->data_ctx.aio_operate(p->oid.name, rados_completion, &op);
        }
+       rados_completion->release();
       }
     }
+    int r = throttle.wait_for_ret();
+    if (r < 0) {
+      lderr(cct) << "warning: failed to remove some object(s): "
+                << cpp_strerror(r) << dendl;
+    }
   }
 
   int read_rbd_info(IoCtx& io_ctx, const string& info_oid,

diff --git a/src/librbd/internal.h b/src/librbd/internal.h
index aae8b7fff0d3bce7acf6bcdb8a394111b1aa156e..4b46594cc913a58321e67d886708982c3e6da505 100644 (file)
--- a/src/librbd/internal.h
+++ b/src/librbd/internal.h
@@ -204,6 +204,7 @@ namespace librbd {
   // raw callbacks
   int simple_read_cb(uint64_t ofs, size_t len, const char *buf, void *arg);
   void rados_req_cb(rados_completion_t cb, void *arg);
+  void rados_ctx_cb(rados_completion_t cb, void *arg);
   void rbd_req_cb(completion_t cb, void *arg);
 }