From: Josh Durgin <josh.durgin@inktank.com>
Date: Tue, 7 May 2013 00:03:34 +0000 (-0700)
Subject: librbd: delete more than one object at once
X-Git-Tag: v0.63~39^2~7
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=40956410169709c32a282d9b872cb5f618a48926;p=ceph.git

librbd: delete more than one object at once

Speed up deletions when resizing down or removing an image by keeping
up 10 operations in flight by default.

Refs: #2256
Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
---

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 84df2bff2666..0a3aa85261e9 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -534,6 +534,7 @@ OPTION(rbd_cache_max_dirty, OPT_LONGLONG, 24<<20)    // dirty limit in bytes - s
 OPTION(rbd_cache_target_dirty, OPT_LONGLONG, 16<<20) // target dirty limit in bytes
 OPTION(rbd_cache_max_dirty_age, OPT_FLOAT, 1.0)      // seconds in cache before writeback starts
 OPTION(rbd_cache_block_writes_upfront, OPT_BOOL, false) // whether to block writes to the cache before the aio_write call completes (true), or block before the aio completion is called (false)
+OPTION(rbd_concurrent_management_ops, OPT_INT, 10) // how many operations can be in flight for a management operation like deleting or resizing an image
 
 OPTION(nss_db_path, OPT_STR, "") // path to nss db
 
diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc
index 8873ac69a403..4d3b55ee0707 100644
--- a/src/librbd/internal.cc
+++ b/src/librbd/internal.cc
@@ -6,6 +6,7 @@
 #include "common/ceph_context.h"
 #include "common/dout.h"
 #include "common/errno.h"
+#include "common/Throttle.h"
 #include "cls/lock/cls_lock_client.h"
 #include "include/inttypes.h"
 #include "include/stringify.h"
@@ -151,22 +152,30 @@ namespace librbd {
     uint64_t period = ictx->get_stripe_period();
     uint64_t num_period = ((newsize + period - 1) / period);
     uint64_t delete_off = MIN(num_period * period, size);
-    uint64_t delete_start = num_period * ictx->get_stripe_count();     // first object we can delete free and clear
+    // first object we can delete free and clear
+    uint64_t delete_start = num_period * ictx->get_stripe_count();
     uint64_t num_objects = ictx->get_num_objects();
     uint64_t object_size = ictx->get_object_size();
 
     ldout(cct, 10) << "trim_image " << size << " -> " << newsize
 		   << " periods " << num_period
 		   << " discard to offset " << delete_off
-		   << " delete objects " << delete_start << " to " << (num_objects-1)
+		   << " delete objects " << delete_start
+		   << " to " << (num_objects-1)
 		   << dendl;
 
+    SimpleThrottle throttle(cct->_conf->rbd_concurrent_management_ops, true);
     if (delete_start < num_objects) {
       ldout(cct, 2) << "trim_image objects " << delete_start << " to "
 		    << (num_objects - 1) << dendl;
       for (uint64_t i = delete_start; i < num_objects; ++i) {
+	throttle.start_op();
 	string oid = ictx->get_object_name(i);
-	ictx->data_ctx.remove(oid);
+	Context *req_comp = new C_SimpleThrottle(&throttle);
+	librados::AioCompletion *rados_completion =
+	  librados::Rados::aio_create_completion(req_comp, NULL, rados_ctx_cb);
+	ictx->data_ctx.aio_remove(oid, rados_completion);
+	rados_completion->release();
 	prog_ctx.update_progress((i - delete_start) * object_size,
 				 (num_objects - delete_start) * object_size);
       }
@@ -175,19 +184,31 @@ namespace librbd {
     // discard the weird boundary, if any
     if (delete_off > newsize) {
       vector<ObjectExtent> extents;
-      Striper::file_to_extents(ictx->cct, ictx->format_string, &ictx->layout, newsize, delete_off - newsize, extents);
+      Striper::file_to_extents(ictx->cct, ictx->format_string, &ictx->layout,
+			       newsize, delete_off - newsize, extents);
 
-      for (vector<ObjectExtent>::iterator p = extents.begin(); p != extents.end(); ++p) {
+      for (vector<ObjectExtent>::iterator p = extents.begin();
+	   p != extents.end(); ++p) {
 	ldout(ictx->cct, 20) << " ex " << *p << dendl;
+	throttle.start_op();
+	Context *req_comp = new C_SimpleThrottle(&throttle);
+	librados::AioCompletion *rados_completion =
+	  librados::Rados::aio_create_completion(req_comp, NULL, rados_ctx_cb);
 	if (p->offset == 0) {
-	  ictx->data_ctx.remove(p->oid.name);
+	  ictx->data_ctx.aio_remove(p->oid.name, rados_completion);
 	} else {
 	  librados::ObjectWriteOperation op;
 	  op.truncate(p->offset);
-	  ictx->data_ctx.operate(p->oid.name, &op);
+	  ictx->data_ctx.aio_operate(p->oid.name, rados_completion, &op);
 	}
+	rados_completion->release();
       }
     }
+    int r = throttle.wait_for_ret();
+    if (r < 0) {
+      lderr(cct) << "warning: failed to remove some object(s): "
+		 << cpp_strerror(r) << dendl;
+    }
   }
 
   int read_rbd_info(IoCtx& io_ctx, const string& info_oid,
diff --git a/src/librbd/internal.h b/src/librbd/internal.h
index aae8b7fff0d3..4b46594cc913 100644
--- a/src/librbd/internal.h
+++ b/src/librbd/internal.h
@@ -204,6 +204,7 @@ namespace librbd {
   // raw callbacks
   int simple_read_cb(uint64_t ofs, size_t len, const char *buf, void *arg);
   void rados_req_cb(rados_completion_t cb, void *arg);
+  void rados_ctx_cb(rados_completion_t cb, void *arg);
   void rbd_req_cb(completion_t cb, void *arg);
 }