From 334631ae4641824b3df49245f36a8fd4b143bf3f Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 29 Aug 2014 19:40:29 -0700
Subject: [PATCH] os/FileJournal: stop aio completion thread *after* writer
 thread

The writer thread may submit a new aio to update the header in its
final moments before shutting down.  Do not stop the aio thread until after
that has happened or else we may not wait for those aio completions.

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit c776a89880fdac270e6334ad8e49fa616d05d0d4)

Conflicts:
	src/os/FileJournal.cc
---
 src/os/FileJournal.cc | 22 +++++++++++++---------
 src/os/FileJournal.h  |  2 ++
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/src/os/FileJournal.cc b/src/os/FileJournal.cc
index 7eb7927e0412a..88f575d74c69e 100644
--- a/src/os/FileJournal.cc
+++ b/src/os/FileJournal.cc
@@ -602,6 +602,7 @@ int FileJournal::dump(ostream& out)
 void FileJournal::start_writer()
 {
   write_stop = false;
+  aio_stop = false;
   write_thread.create();
 #ifdef HAVE_LIBAIO
   write_finish_thread.create();
@@ -612,20 +613,23 @@ void FileJournal::stop_writer()
 {
   {
     Mutex::Locker l(write_lock);
-#ifdef HAVE_LIBAIO
-    Mutex::Locker q(aio_lock);
-#endif
     Mutex::Locker p(writeq_lock);
     write_stop = true;
     writeq_cond.Signal();
+  }
+  write_thread.join();
+
 #ifdef HAVE_LIBAIO
+  // stop aio completeion thread *after* writer thread has stopped
+  // and has submitted all of its io
+  if (aio) {
+    aio_lock.Lock();
+    aio_stop = true;
     aio_cond.Signal();
     write_finish_cond.Signal();
-#endif
-  } 
-  write_thread.join();
-#ifdef HAVE_LIBAIO
-  write_finish_thread.join();
+    aio_lock.Unlock();
+    write_finish_thread.join();
+  }
 #endif
 }
 
@@ -1327,7 +1331,7 @@ void FileJournal::write_finish_thread_entry()
     {
       Mutex::Locker locker(aio_lock);
       if (aio_queue.empty()) {
-	if (write_stop)
+	if (aio_stop)
 	  break;
 	dout(20) << "write_finish_thread_entry sleeping" << dendl;
 	write_finish_cond.Wait(aio_lock);
diff --git a/src/os/FileJournal.h b/src/os/FileJournal.h
index dbb1181bc8ed6..31cf289ed13f5 100644
--- a/src/os/FileJournal.h
+++ b/src/os/FileJournal.h
@@ -288,6 +288,7 @@ private:
   // write thread
   Mutex write_lock;
   bool write_stop;
+  bool aio_stop;
 
   Cond commit_cond;
 
@@ -379,6 +380,7 @@ private:
     throttle_bytes(g_ceph_context, "filestore_bytes"),
     write_lock("FileJournal::write_lock", false, true, false, g_ceph_context),
     write_stop(false),
+    aio_stop(false),
     write_thread(this),
     write_finish_thread(this) { }
   ~FileJournal() {
-- 
2.39.5