From 75ce8000b60411c9e552a9c4363f73fa80ba1fca Mon Sep 17 00:00:00 2001
From: Yehuda Sadeh <yehuda@redhat.com>
Date: Wed, 18 Oct 2017 17:39:34 -0700
Subject: [PATCH] rgw: fix io blocked accounting

Now that we can have multiple IOs per coroutine, need to only decrease
num_blocked one time when switching from io blocked to unblocked. Also,
is_blocked() should not return true if is_done.

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
---
 src/rgw/rgw_coroutine.cc | 13 +++++++++----
 src/rgw/rgw_coroutine.h  |  2 +-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/rgw/rgw_coroutine.cc b/src/rgw/rgw_coroutine.cc
index 4fdf59aa3f036..f4e1e0ab1dd07 100644
--- a/src/rgw/rgw_coroutine.cc
+++ b/src/rgw/rgw_coroutine.cc
@@ -224,6 +224,7 @@ int RGWCoroutinesStack::operate(RGWCoroutinesEnv *_env)
     r = unwind(op_retcode);
     op->put();
     done_flag = (pos == ops.end());
+    blocked_flag &= ~done_flag;
     if (done_flag) {
       retcode = op_retcode;
     }
@@ -321,6 +322,7 @@ int RGWCoroutinesStack::unwind(int retcode)
   rgw_spawned_stacks *src_spawned = &(*pos)->spawned;
 
   if (pos == ops.begin()) {
+    ldout(cct, 0) << "stack " << (void *)this << " end" << dendl;
     spawned.inherit(src_spawned);
     ops.clear();
     pos = ops.end();
@@ -505,11 +507,14 @@ void RGWCoroutinesManager::handle_unblocked_stack(set<RGWCoroutinesStack *>& con
   if (!stack->try_io_unblock(io.io_id)) {
     return;
   }
-  --(*blocked_count);
-  stack->set_io_blocked(false);
+  if (stack->is_io_blocked()) {
+    --(*blocked_count);
+    stack->set_io_blocked(false);
+  }
   stack->set_interval_wait(false);
   if (!stack->is_done()) {
     scheduled_stacks.push_back(stack);
+    stack->set_is_scheduled(true);
   } else {
     context_stacks.erase(stack);
     stack->put();
@@ -651,7 +656,7 @@ int RGWCoroutinesManager::run(list<RGWCoroutinesStack *>& stacks)
       ret = completion_mgr->get_next(&io);
       lock.get_write();
       if (ret < 0) {
-       ldout(cct, 0) << "ERROR: failed to clone shard, completion_mgr.get_next() returned ret=" << ret << dendl;
+       ldout(cct, 0) << "ERROR: completion_mgr.get_next() returned ret=" << ret << dendl;
       }
       handle_unblocked_stack(context_stacks, scheduled_stacks, io, &blocked_count);
     }
@@ -666,7 +671,7 @@ next:
       ret = completion_mgr->get_next(&io);
       lock.get_write();
       if (ret < 0) {
-	ldout(cct, 0) << "ERROR: failed to clone shard, completion_mgr.get_next() returned ret=" << ret << dendl;
+	ldout(cct, 0) << "ERROR: completion_mgr.get_next() returned ret=" << ret << dendl;
       }
       if (going_down) {
 	ldout(cct, 5) << __func__ << "(): was stopped, exiting" << dendl;
diff --git a/src/rgw/rgw_coroutine.h b/src/rgw/rgw_coroutine.h
index d83ccb55916ae..25dc55660ea7d 100644
--- a/src/rgw/rgw_coroutine.h
+++ b/src/rgw/rgw_coroutine.h
@@ -413,7 +413,7 @@ public:
     io_blocked_id = io_id;
   }
   bool is_io_blocked() {
-    return blocked_flag;
+    return blocked_flag && !done_flag;
   }
   bool can_io_unblock(int64_t io_id) {
     return (io_blocked_id == io_id) ||
-- 
2.39.5