From 77f7c1cf829e5417eb356ebf3bf1fbc8ff7a25a5 Mon Sep 17 00:00:00 2001
From: Hector Martin <marcan@marcan.st>
Date: Wed, 25 May 2016 20:41:40 +0900
Subject: [PATCH] librbd: only mark async completion as done after callback

This change in behavior was introduced in fde9f78.

Signed-off-by: Hector Martin <marcan@marcan.st>
---
 src/librbd/AioCompletion.cc | 12 +++++++-----
 src/librbd/AioCompletion.h  | 10 ++++++++--
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/librbd/AioCompletion.cc b/src/librbd/AioCompletion.cc
index 3501032e80893..e439aaaf2efb1 100644
--- a/src/librbd/AioCompletion.cc
+++ b/src/librbd/AioCompletion.cc
@@ -31,7 +31,7 @@ namespace librbd {
   int AioCompletion::wait_for_complete() {
     tracepoint(librbd, aio_wait_for_complete_enter, this);
     lock.Lock();
-    while (!done)
+    while (state != STATE_COMPLETE)
       cond.Wait(lock);
     lock.Unlock();
     tracepoint(librbd, aio_wait_for_complete_exit, 0);
@@ -95,7 +95,7 @@ namespace librbd {
       ictx->journal->commit_io_event(journal_tid, rval);
     }
 
-    done = true;
+    state = STATE_CALLBACK;
     if (complete_cb) {
       lock.Unlock();
       complete_cb(rbd_comp, complete_arg);
@@ -108,6 +108,8 @@ namespace librbd {
       ictx->completed_reqs_lock.Unlock();
       ictx->event_socket.notify();
     }
+
+    state = STATE_COMPLETE;
     cond.Signal();
 
     // note: possible for image to be closed after op marked finished
@@ -129,7 +131,7 @@ namespace librbd {
     init_time(i, t);
 
     Mutex::Locker locker(lock);
-    if (!done && !async_op.started()) {
+    if (state == STATE_PENDING && !async_op.started()) {
       async_op.start_op(*ictx);
     }
   }
@@ -179,7 +181,7 @@ namespace librbd {
 
   void AioCompletion::associate_journal_event(uint64_t tid) {
     Mutex::Locker l(lock);
-    assert(!done);
+    assert(state == STATE_PENDING);
     journal_tid = tid;
   }
 
@@ -188,7 +190,7 @@ namespace librbd {
     bool done;
     {
       Mutex::Locker l(lock);
-      done = this->done;
+      done = this->state == STATE_COMPLETE;
     }
     tracepoint(librbd, aio_is_complete_exit, done);
     return done;
diff --git a/src/librbd/AioCompletion.h b/src/librbd/AioCompletion.h
index c554bc5072ef6..6c37f91d55d58 100644
--- a/src/librbd/AioCompletion.h
+++ b/src/librbd/AioCompletion.h
@@ -30,6 +30,12 @@ namespace librbd {
     AIO_TYPE_FLUSH,
   } aio_type_t;
 
+  typedef enum {
+    STATE_PENDING = 0,
+    STATE_CALLBACK,
+    STATE_COMPLETE,
+  } aio_state_t;
+
   /**
    * AioCompletion is the overall completion for a single
    * rbd I/O request. It may be composed of many AioObjectRequests,
@@ -46,7 +52,7 @@ namespace librbd {
   struct AioCompletion {
     Mutex lock;
     Cond cond;
-    bool done;
+    aio_state_t state;
     ssize_t rval;
     callback_t complete_cb;
     void *complete_arg;
@@ -95,7 +101,7 @@ namespace librbd {
     }
 
     AioCompletion() : lock("AioCompletion::lock", true, false),
-		      done(false), rval(0), complete_cb(NULL),
+		      state(STATE_PENDING), rval(0), complete_cb(NULL),
 		      complete_arg(NULL), rbd_comp(NULL),
 		      pending_count(0), blockers(1),
 		      ref(1), released(false), ictx(NULL),
-- 
2.39.5