rgw: trim gc index using aio

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
(cherry picked from commit f00ee63e981646d0dc19bcb3d1e4165ef0a8cfce)
Signed-off-by: Jesse Williamson <jesse.williamson@canonical.com>
Conflicts: rgw_gc.cc (uses a kind of aggregate initialization not available in C++11)

diff --git a/src/rgw/rgw_gc.cc b/src/rgw/rgw_gc.cc
index 9ff04b09058b16413a581d8b349af55dafaa6a86..252254698a03e6390dc94b6614acaf4d942cc19b 100644 (file)
--- a/src/rgw/rgw_gc.cc
+++ b/src/rgw/rgw_gc.cc
@@ -81,11 +81,11 @@ int RGWGC::defer_chain(const string& tag, bool sync)
   return store->gc_aio_operate(obj_names[i], &op);
 }
 
-int RGWGC::remove(int index, const std::list<string>& tags)
+int RGWGC::remove(int index, const std::list<string>& tags, AioCompletion **pc)
 {
   ObjectWriteOperation op;
   cls_rgw_gc_remove(op, tags);
-  return store->gc_operate(obj_names[index], &op);
+  return store->gc_aio_operate(obj_names[index], &op, pc);
 }
 
 int RGWGC::list(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated)
@@ -133,10 +133,22 @@ class RGWGCIOManager {
   RGWGC *gc;
 
   struct IO {
+    enum Type {
+      UnknownIO = 0,
+      TailIO = 1,
+      IndexIO = 2,
+    } type{UnknownIO};
     librados::AioCompletion *c{nullptr};
     string oid;
     int index{-1};
     string tag;
+
+    IO() = default;
+
+    IO(IO::Type type, librados::AioCompletion *c, 
+       const std::string& oid, int index, const std::string& tag)
+     : type(type), c(c), oid(oid), index(index), tag(tag)
+    {}
   };
 
   list<IO> ios;
@@ -166,7 +178,7 @@ public:
     if (ret < 0) {
       return ret;
     }
-    ios.push_back(IO{c, oid, index, tag});
+    ios.push_back(IO(IO::TailIO, c, oid, index, tag));
 
     return 0;
   }
@@ -183,6 +195,14 @@ public:
     if (ret == -ENOENT) {
       ret = 0;
     }
+
+    if (io.type == IO::IndexIO) {
+      if (ret < 0) {
+        ldout(cct, 0) << "WARNING: gc cleanup of tags on gc shard index=" << io.index << " returned error, ret=" << ret << dendl;
+      }
+      goto done;
+    }
+
     if (ret < 0) {
       ldout(cct, 0) << "WARNING: could not remove oid=" << io.oid << ", ret=" << ret << dendl;
       goto done;
@@ -191,31 +211,48 @@ public:
     rt.push_back(io.tag);
 #define MAX_REMOVE_CHUNK 16
     if (rt.size() > MAX_REMOVE_CHUNK) {
-      drain_remove_tags(io.index, rt);
+      flush_remove_tags(io.index, rt);
     }
 done:
     ios.pop_front();
   }
 
-  void drain() {
+  void drain_ios() {
     while (!ios.empty()) {
       if (gc->going_down()) {
         return;
       }
       handle_next_completion();
     }
+  }
 
-    drain_remove_tags();
+  void drain() {
+    drain_ios();
+    flush_remove_tags();
+    /* the tags draining might have generated more ios, drain those too */
+    drain_ios();
   }
 
-  void drain_remove_tags(int index, list<string>& rt) {
-    gc->remove(index, rt);
+  void flush_remove_tags(int index, list<string>& rt) {
+    IO index_io;
+    index_io.type = IO::IndexIO;
+    index_io.index = index;
+
+    int ret = gc->remove(index, rt, &index_io.c);
     rt.clear();
+    if (ret < 0) {
+      /* we already cleared list of tags, this prevents us from ballooning in case of
+       * a persistent problem
+       */
+      ldout(cct, 0) << "WARNING: failed to remove tags on gc shard index=" << index << " ret=" << ret << dendl;
+      return;
+    }
+    ios.push_back(index_io);
   }
 
-  void drain_remove_tags() {
+  void flush_remove_tags() {
     for (auto iter : remove_tags) {
-      drain_remove_tags(iter.first, iter.second);
+      flush_remove_tags(iter.first, iter.second);
     }
   }
 };

diff --git a/src/rgw/rgw_gc.h b/src/rgw/rgw_gc.h
index 7eb38711db41c55c8d1d0a327e8d2b2c8410c147..e901888ee8fb18739ae838b909f5f17642f5fa40 100644 (file)
--- a/src/rgw/rgw_gc.h
+++ b/src/rgw/rgw_gc.h
@@ -50,7 +50,7 @@ public:
   void add_chain(librados::ObjectWriteOperation& op, cls_rgw_obj_chain& chain, const string& tag);
   int send_chain(cls_rgw_obj_chain& chain, const string& tag, bool sync);
   int defer_chain(const string& tag, bool sync);
-  int remove(int index, const std::list<string>& tags);
+  int remove(int index, const std::list<string>& tags, librados::AioCompletion **pc);
 
   void initialize(CephContext *_cct, RGWRados *_store);
   void finalize();

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index e578d7141fdb1f96b1eac662663d0b0465d755a1..9219e4f260bc14132d9f4ced1d4e3c2ac2e5c1ce 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -13194,11 +13194,15 @@ int RGWRados::gc_operate(string& oid, librados::ObjectWriteOperation *op)
   return gc_pool_ctx.operate(oid, op);
 }
 
-int RGWRados::gc_aio_operate(string& oid, librados::ObjectWriteOperation *op)
+int RGWRados::gc_aio_operate(string& oid, librados::ObjectWriteOperation *op, AioCompletion **pc)
 {
   AioCompletion *c = librados::Rados::aio_create_completion(NULL, NULL, NULL);
   int r = gc_pool_ctx.aio_operate(oid, c, op);
-  c->release();
+  if (!pc) {
+    c->release();
+  } else {
+    *pc = c;
+  }
   return r;
 }
 

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 2fbb27c9fb10728e67a4ac39987ac33d10f06bb6..06a2c4a649ef0b61a9177f0742c42131a429c2e8 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -3620,7 +3620,7 @@ public:
   void update_gc_chain(rgw_obj& head_obj, RGWObjManifest& manifest, cls_rgw_obj_chain *chain);
   int send_chain_to_gc(cls_rgw_obj_chain& chain, const string& tag, bool sync);
   int gc_operate(string& oid, librados::ObjectWriteOperation *op);
-  int gc_aio_operate(string& oid, librados::ObjectWriteOperation *op);
+  int gc_aio_operate(string& oid, librados::ObjectWriteOperation *op, librados::AioCompletion **pc = nullptr);
   int gc_operate(string& oid, librados::ObjectReadOperation *op, bufferlist *pbl);
 
   int list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated);