rgw: trim gc index using aio

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/rgw/rgw_gc.cc b/src/rgw/rgw_gc.cc
index 0e5a0725f3430f7b8c0b92ca6d35d71cad3505ce..6cccf62e7586befa06a1a25a8f4e4eeb51ab71d7 100644 (file)
--- a/src/rgw/rgw_gc.cc
+++ b/src/rgw/rgw_gc.cc
@@ -80,11 +80,11 @@ int RGWGC::defer_chain(const string& tag, bool sync)
   return store->gc_aio_operate(obj_names[i], &op);
 }
 
-int RGWGC::remove(int index, const std::list<string>& tags)
+int RGWGC::remove(int index, const std::list<string>& tags, AioCompletion **pc)
 {
   ObjectWriteOperation op;
   cls_rgw_gc_remove(op, tags);
-  return store->gc_operate(obj_names[index], &op);
+  return store->gc_aio_operate(obj_names[index], &op, pc);
 }
 
 int RGWGC::list(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated)
@@ -132,6 +132,11 @@ class RGWGCIOManager {
   RGWGC *gc;
 
   struct IO {
+    enum Type {
+      UnknownIO = 0,
+      TailIO = 1,
+      IndexIO = 2,
+    } type{UnknownIO};
     librados::AioCompletion *c{nullptr};
     string oid;
     int index{-1};
@@ -165,7 +170,7 @@ public:
     if (ret < 0) {
       return ret;
     }
-    ios.push_back(IO{c, oid, index, tag});
+    ios.push_back(IO{IO::TailIO, c, oid, index, tag});
 
     return 0;
   }
@@ -182,6 +187,14 @@ public:
     if (ret == -ENOENT) {
       ret = 0;
     }
+
+    if (io.type == IO::IndexIO) {
+      if (ret < 0) {
+        ldout(cct, 0) << "WARNING: gc cleanup of tags on gc shard index=" << io.index << " returned error, ret=" << ret << dendl;
+      }
+      goto done;
+    }
+
     if (ret < 0) {
       ldout(cct, 0) << "WARNING: could not remove oid=" << io.oid << ", ret=" << ret << dendl;
       goto done;
@@ -190,31 +203,48 @@ public:
     rt.push_back(io.tag);
 #define MAX_REMOVE_CHUNK 16
     if (rt.size() > MAX_REMOVE_CHUNK) {
-      drain_remove_tags(io.index, rt);
+      flush_remove_tags(io.index, rt);
     }
 done:
     ios.pop_front();
   }
 
-  void drain() {
+  void drain_ios() {
     while (!ios.empty()) {
       if (gc->going_down()) {
         return;
       }
       handle_next_completion();
     }
+  }
 
-    drain_remove_tags();
+  void drain() {
+    drain_ios();
+    flush_remove_tags();
+    /* the tags draining might have generated more ios, drain those too */
+    drain_ios();
   }
 
-  void drain_remove_tags(int index, list<string>& rt) {
-    gc->remove(index, rt);
+  void flush_remove_tags(int index, list<string>& rt) {
+    IO index_io;
+    index_io.type = IO::IndexIO;
+    index_io.index = index;
+
+    int ret = gc->remove(index, rt, &index_io.c);
     rt.clear();
+    if (ret < 0) {
+      /* we already cleared list of tags, this prevents us from ballooning in case of
+       * a persistent problem
+       */
+      ldout(cct, 0) << "WARNING: failed to remove tags on gc shard index=" << index << " ret=" << ret << dendl;
+      return;
+    }
+    ios.push_back(index_io);
   }
 
-  void drain_remove_tags() {
+  void flush_remove_tags() {
     for (auto iter : remove_tags) {
-      drain_remove_tags(iter.first, iter.second);
+      flush_remove_tags(iter.first, iter.second);
     }
   }
 };

diff --git a/src/rgw/rgw_gc.h b/src/rgw/rgw_gc.h
index 7eb38711db41c55c8d1d0a327e8d2b2c8410c147..e901888ee8fb18739ae838b909f5f17642f5fa40 100644 (file)
--- a/src/rgw/rgw_gc.h
+++ b/src/rgw/rgw_gc.h
@@ -50,7 +50,7 @@ public:
   void add_chain(librados::ObjectWriteOperation& op, cls_rgw_obj_chain& chain, const string& tag);
   int send_chain(cls_rgw_obj_chain& chain, const string& tag, bool sync);
   int defer_chain(const string& tag, bool sync);
-  int remove(int index, const std::list<string>& tags);
+  int remove(int index, const std::list<string>& tags, librados::AioCompletion **pc);
 
   void initialize(CephContext *_cct, RGWRados *_store);
   void finalize();

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 9061426b4317628864c2bfc3b74b5bacc3767c4b..28751de49cf10ad6969b388eb0c8e5f4e163efb2 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -12753,11 +12753,15 @@ int RGWRados::gc_operate(string& oid, librados::ObjectWriteOperation *op)
   return gc_pool_ctx.operate(oid, op);
 }
 
-int RGWRados::gc_aio_operate(string& oid, librados::ObjectWriteOperation *op)
+int RGWRados::gc_aio_operate(string& oid, librados::ObjectWriteOperation *op, AioCompletion **pc)
 {
   AioCompletion *c = librados::Rados::aio_create_completion(NULL, NULL, NULL);
   int r = gc_pool_ctx.aio_operate(oid, c, op);
-  c->release();
+  if (!pc) {
+    c->release();
+  } else {
+    *pc = c;
+  }
   return r;
 }
 

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index c77639381285f6eb19bcba08464ba469ecc68b64..d02a81b4b864987763712ce9fdc2463b57680e7c 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -3587,7 +3587,7 @@ public:
   void update_gc_chain(rgw_obj& head_obj, RGWObjManifest& manifest, cls_rgw_obj_chain *chain);
   int send_chain_to_gc(cls_rgw_obj_chain& chain, const string& tag, bool sync);
   int gc_operate(string& oid, librados::ObjectWriteOperation *op);
-  int gc_aio_operate(string& oid, librados::ObjectWriteOperation *op);
+  int gc_aio_operate(string& oid, librados::ObjectWriteOperation *op, librados::AioCompletion **pc = nullptr);
   int gc_operate(string& oid, librados::ObjectReadOperation *op, bufferlist *pbl);
 
   int list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated);