From e0de824f7e728b06025a572a0c8213ef9fb1f112 Mon Sep 17 00:00:00 2001
From: Jason Dillaman <dillaman@redhat.com>
Date: Wed, 21 Sep 2016 15:41:55 -0400
Subject: [PATCH] rbd-mirror: wait for in-flight event commit notifications

Fixes: http://tracker.ceph.com/issues/17355
Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit f810c106ad4e9ae94494801fea6c580d81b9156b)
---
 src/tools/rbd_mirror/ImageReplayer.cc | 14 ++++++++------
 src/tools/rbd_mirror/ImageReplayer.h  |  2 ++
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc
index 2ad1bd0189697..923c4981d57b2 100644
--- a/src/tools/rbd_mirror/ImageReplayer.cc
+++ b/src/tools/rbd_mirror/ImageReplayer.cc
@@ -1050,6 +1050,8 @@ void ImageReplayer<I>::process_entry() {
   Context *on_ready = create_context_callback<
     ImageReplayer, &ImageReplayer<I>::handle_process_entry_ready>(this);
   Context *on_commit = new C_ReplayCommitted(this, std::move(m_replay_entry));
+
+  m_event_replay_tracker.start_op();
   m_local_replay->process(m_event_entry, on_ready, on_commit);
 }
 
@@ -1070,14 +1072,12 @@ void ImageReplayer<I>::handle_process_entry_safe(const ReplayEntry& replay_entry
 
   if (r < 0) {
     derr << "failed to commit journal event: " << cpp_strerror(r) << dendl;
-
     handle_replay_complete(r, "failed to commit journal event");
-    return;
-  }
-
-  if (m_remote_journaler) {
+  } else {
+    assert(m_remote_journaler != nullptr);
     m_remote_journaler->committed(replay_entry);
   }
+  m_event_replay_tracker.finish_op();
 }
 
 template <typename I>
@@ -1374,7 +1374,9 @@ void ImageReplayer<I>::shut_down(int r) {
 
         // blocks if listener notification is in-progress
         m_local_journal->remove_listener(m_journal_listener);
-        ctx->complete(0);
+
+        // wait for all in-flight replayed events to complete
+        m_event_replay_tracker.wait_for_ops(ctx);
       });
     if (m_local_replay != nullptr) {
       ctx = new FunctionContext([this, ctx](int r) {
diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h
index 3765dc0867064..222ef024a5e4e 100644
--- a/src/tools/rbd_mirror/ImageReplayer.h
+++ b/src/tools/rbd_mirror/ImageReplayer.h
@@ -9,6 +9,7 @@
 #include <vector>
 
 #include "include/atomic.h"
+#include "common/AsyncOpTracker.h"
 #include "common/Mutex.h"
 #include "common/WorkQueue.h"
 #include "include/rados/librados.hpp"
@@ -289,6 +290,7 @@ private:
   cls::journal::Tag m_replay_tag;
   librbd::journal::TagData m_replay_tag_data;
   librbd::journal::EventEntry m_event_entry;
+  AsyncOpTracker m_event_replay_tracker;
 
   struct RemoteJournalerListener : public ::journal::JournalMetadataListener {
     ImageReplayer *replayer;
-- 
2.39.5