From aa829e399fe94458468f41d56730b99bf42b9b03 Mon Sep 17 00:00:00 2001
From: Yehuda Sadeh <yehuda@redhat.com>
Date: Tue, 29 Sep 2015 14:24:28 -0700
Subject: [PATCH] rgw: clean up child draining a bit

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
---
 src/rgw/rgw_coroutine.cc | 19 +++++++++++++++++++
 src/rgw/rgw_coroutine.h  | 18 +++++++++++++++++-
 src/rgw/rgw_data_sync.cc | 22 +++-------------------
 3 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/src/rgw/rgw_coroutine.cc b/src/rgw/rgw_coroutine.cc
index 73e6bc96077ac..ae5a2cd9d696b 100644
--- a/src/rgw/rgw_coroutine.cc
+++ b/src/rgw/rgw_coroutine.cc
@@ -449,6 +449,25 @@ void RGWCoroutine::wait_for_child()
   }
 }
 
+bool RGWCoroutine::drain_children()
+{
+  bool done = false;
+  reenter(drain_cr) {
+    yield wait_for_child();
+    int ret;
+    while (collect(&ret)) {
+      if (ret < 0) {
+        ldout(cct, 0) << "ERROR: a sync operation returned error" << dendl;
+        /* we should have reported this error */
+#warning deal with error
+      }
+      yield wait_for_child();
+    }
+    done = true;
+  }
+  return !done;
+}
+
 void RGWCoroutine::wakeup()
 {
   stack->wakeup();
diff --git a/src/rgw/rgw_coroutine.h b/src/rgw/rgw_coroutine.h
index f254604711156..0e949c63505a2 100644
--- a/src/rgw/rgw_coroutine.h
+++ b/src/rgw/rgw_coroutine.h
@@ -125,10 +125,14 @@ struct rgw_spawned_stacks {
   }
 };
 
+
 class RGWCoroutine : public RefCountedObject, public boost::asio::coroutine {
   friend class RGWCoroutinesStack;
 
 protected:
+  bool _yield_ret;
+  boost::asio::coroutine drain_cr;
+
   CephContext *cct;
 
   RGWCoroutinesStack *stack;
@@ -148,7 +152,7 @@ protected:
   int io_block(int ret);
 
 public:
-  RGWCoroutine(CephContext *_cct) : cct(_cct), stack(NULL), retcode(0), state(RGWCoroutine_Run) {}
+  RGWCoroutine(CephContext *_cct) : _yield_ret(false), cct(_cct), stack(NULL), retcode(0), state(RGWCoroutine_Run) {}
   virtual ~RGWCoroutine() {}
 
   virtual int operate() = 0;
@@ -174,6 +178,7 @@ public:
   bool collect(int *ret); /* returns true if needs to be called again */
 
   int wait(const utime_t& interval);
+  bool drain_children(); /* returns true if needed to be called again */
   void wakeup();
 
   size_t num_spawned() {
@@ -183,6 +188,17 @@ public:
   void wait_for_child();
 };
 
+#define yield_until_true(x)     \
+do {                            \
+  do {                          \
+    yield _yield_ret = x;       \
+  } while (!_yield_ret);        \
+  _yield_ret = false;           \
+} while (0)
+
+#define drain_all() \
+  yield_until_true(drain_children())
+
 template <class T>
 class RGWConsumerCR : public RGWCoroutine {
   list<T> product;
diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc
index 536246afedb5b..60c4533b87cc2 100644
--- a/src/rgw/rgw_data_sync.cc
+++ b/src/rgw/rgw_data_sync.cc
@@ -1330,15 +1330,7 @@ int RGWBucketShardIncrementalSyncCR::operate()
       }
       if (retcode < 0 && retcode != -ENOENT) {
         /* wait for all operations to complete */
-        yield wait_for_child();
-        while (collect(&ret)) {
-          if (ret < 0) {
-            ldout(store->ctx(), 0) << "ERROR: a sync operation returned error" << dendl;
-            /* we should have reported this error */
-#warning deal with error
-          }
-          yield wait_for_child();
-        }
+        drain_all();
         return set_state(RGWCoroutine_Error, retcode);
       }
       entries_iter = list_result.begin();
@@ -1364,22 +1356,14 @@ int RGWBucketShardIncrementalSyncCR::operate()
               /* we should have reported this error */
 #warning deal with error
             }
-            yield wait_for_child();
+            /* not waiting for child here */
           }
         }
       }
     } while (!list_result.empty());
 
     /* wait for all operations to complete */
-    yield wait_for_child();
-    while (collect(&ret)) {
-      if (ret < 0) {
-        ldout(store->ctx(), 0) << "ERROR: a sync operation returned error" << dendl;
-        /* we should have reported this error */
-#warning deal with error
-      }
-      yield wait_for_child();
-    }
+    drain_all();
     return set_state(RGWCoroutine_Done, 0);
   }
   return 0;
-- 
2.39.5