rgw: use a single gc io manager for all shards

to allow cross shards concurrency

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
(cherry picked from commit e60404cb18682dcc6dad03ee1806a601d5c8b2e0)

diff --git a/src/rgw/rgw_gc.cc b/src/rgw/rgw_gc.cc
index 00f82aa2831514ea5bf9ebf155aa9b8a6fbee052..9ff04b09058b16413a581d8b349af55dafaa6a86 100644 (file)
--- a/src/rgw/rgw_gc.cc
+++ b/src/rgw/rgw_gc.cc
@@ -131,28 +131,27 @@ int RGWGC::list(int *index, string& marker, uint32_t max, bool expired_only, std
 class RGWGCIOManager {
   CephContext *cct;
   RGWGC *gc;
-  int index;
 
   struct IO {
     librados::AioCompletion *c{nullptr};
     string oid;
+    int index{-1};
     string tag;
   };
 
   list<IO> ios;
-  std::list<string> remove_tags;
+  map<int, std::list<string> > remove_tags;
 
 public:
-  RGWGCIOManager(CephContext *_cct, RGWGC *_gc, int _index) : cct(_cct),
-                                                              gc(_gc),
-                                                              index(_index) {}
+  RGWGCIOManager(CephContext *_cct, RGWGC *_gc) : cct(_cct),
+                                                  gc(_gc) {}
   ~RGWGCIOManager() {
     for (auto io : ios) {
       io.c->release();
     }
   }
 
-  int schedule_io(IoCtx *ioctx, const string& oid, ObjectWriteOperation *op, const string& tag) {
+  int schedule_io(IoCtx *ioctx, const string& oid, ObjectWriteOperation *op, int index, const string& tag) {
 #warning configurable
 #define MAX_CONCURRENT_IO 5
     while (ios.size() > MAX_CONCURRENT_IO) {
@@ -167,7 +166,7 @@ public:
     if (ret < 0) {
       return ret;
     }
-    ios.push_back(IO{c, oid, tag});
+    ios.push_back(IO{c, oid, index, tag});
 
     return 0;
   }
@@ -179,6 +178,8 @@ public:
     int ret = io.c->get_return_value();
     io.c->release();
 
+    auto& rt = remove_tags[io.index];
+
     if (ret == -ENOENT) {
       ret = 0;
     }
@@ -187,10 +188,10 @@ public:
       goto done;
     }
 
-    remove_tags.push_back(io.tag);
+    rt.push_back(io.tag);
 #define MAX_REMOVE_CHUNK 16
-    if (remove_tags.size() > MAX_REMOVE_CHUNK) {
-      drain_remove_tags();
+    if (rt.size() > MAX_REMOVE_CHUNK) {
+      drain_remove_tags(io.index, rt);
     }
 done:
     ios.pop_front();
@@ -207,19 +208,24 @@ done:
     drain_remove_tags();
   }
 
+  void drain_remove_tags(int index, list<string>& rt) {
+    gc->remove(index, rt);
+    rt.clear();
+  }
+
   void drain_remove_tags() {
-    gc->remove(index, remove_tags);
-    remove_tags.clear();
+    for (auto iter : remove_tags) {
+      drain_remove_tags(iter.first, iter.second);
+    }
   }
 };
 
-int RGWGC::process(int index, int max_secs, bool expired_only)
+int RGWGC::process(int index, int max_secs, bool expired_only,
+                   RGWGCIOManager& io_manager)
 {
   rados::cls::lock::Lock l(gc_index_lock_name);
   utime_t end = ceph_clock_now();
 
-  RGWGCIOManager io_manager(store->ctx(), this, index);
-
   /* max_secs should be greater than zero. We don't want a zero max_secs
    * to be translated as no timeout, since we'd then need to break the
    * lock and that would require a manual intervention. In this case
@@ -288,7 +294,7 @@ int RGWGC::process(int index, int max_secs, bool expired_only)
        ObjectWriteOperation op;
        cls_refcount_put(op, info.tag, true);
 
-        ret = io_manager.schedule_io(ctx, oid, &op, info.tag);
+        ret = io_manager.schedule_io(ctx, oid, &op, index, info.tag);
         if (ret < 0) {
           ldout(store->ctx(), 0) << "WARNING: failed to schedule deletion for oid=" << oid << dendl;
         }
@@ -299,8 +305,6 @@ int RGWGC::process(int index, int max_secs, bool expired_only)
     }
   } while (truncated);
 
-  io_manager.drain();
-
 done:
   /* we don't drain here, because if we're going down we don't want to hold the system
    * if backend is unresponsive
@@ -319,12 +323,17 @@ int RGWGC::process(bool expired_only)
   if (ret < 0)
     return ret;
 
+  RGWGCIOManager io_manager(store->ctx(), this);
+
   for (int i = 0; i < max_objs; i++) {
     int index = (i + start) % max_objs;
-    int ret = process(index, max_secs, expired_only);
+    int ret = process(index, max_secs, expired_only, io_manager);
     if (ret < 0)
       return ret;
   }
+  if (!going_down()) {
+    io_manager.drain();
+  }
 
   return 0;
 }

diff --git a/src/rgw/rgw_gc.h b/src/rgw/rgw_gc.h
index bab44b7fd34c9778e3e26542690e5d45a8fbe41c..7eb38711db41c55c8d1d0a327e8d2b2c8410c147 100644 (file)
--- a/src/rgw/rgw_gc.h
+++ b/src/rgw/rgw_gc.h
@@ -16,6 +16,8 @@
 
 #include <atomic>
 
+class RGWGCIOManager;
+
 class RGWGC {
   CephContext *cct;
   RGWRados *store;
@@ -55,7 +57,8 @@ public:
 
   int list(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated);
   void list_init(int *index) { *index = 0; }
-  int process(int index, int process_max_secs, bool expired_only);
+  int process(int index, int process_max_secs, bool expired_only,
+              RGWGCIOManager& io_manager);
   int process(bool expired_only);
 
   bool going_down();