os/FileJournal: stop aio completion thread *after* writer thread

The writer thread may submit a new aio to update the header in its
final moments before shutting down.  Do not stop the aio thread until after
that has happened or else we may not wait for those aio completions.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/os/FileJournal.cc b/src/os/FileJournal.cc
index d6ddc7fa21297c737b15e4d215e1b2e57cc2dd84..c47f9b463bd5926ddf804e06ed76cfaf9434cb5d 100644 (file)
--- a/src/os/FileJournal.cc
+++ b/src/os/FileJournal.cc
@@ -605,6 +605,7 @@ int FileJournal::dump(ostream& out)
 void FileJournal::start_writer()
 {
   write_stop = false;
+  aio_stop = false;
   write_thread.create();
 #ifdef HAVE_LIBAIO
   if (aio)
@@ -616,24 +617,21 @@ void FileJournal::stop_writer()
 {
   {
     Mutex::Locker l(write_lock);
-#ifdef HAVE_LIBAIO
-    if (aio)
-      aio_lock.Lock();
-#endif
     Mutex::Locker p(writeq_lock);
     write_stop = true;
     writeq_cond.Signal();
-#ifdef HAVE_LIBAIO
-    if (aio) {
-      aio_cond.Signal();
-      write_finish_cond.Signal();
-      aio_lock.Unlock();
-    }
-#endif
-  } 
+  }
   write_thread.join();
+
 #ifdef HAVE_LIBAIO
+  // stop aio completeion thread *after* writer thread has stopped
+  // and has submitted all of its io
   if (aio) {
+    aio_lock.Lock();
+    aio_stop = true;
+    aio_cond.Signal();
+    write_finish_cond.Signal();
+    aio_lock.Unlock();
     write_finish_thread.join();
   }
 #endif
@@ -1341,7 +1339,7 @@ void FileJournal::write_finish_thread_entry()
     {
       Mutex::Locker locker(aio_lock);
       if (aio_queue.empty()) {
-       if (write_stop)
+       if (aio_stop)
          break;
        dout(20) << "write_finish_thread_entry sleeping" << dendl;
        write_finish_cond.Wait(aio_lock);

diff --git a/src/os/FileJournal.h b/src/os/FileJournal.h
index 94197aab3dc94294685911d81e86d7be2c2801e2..4124ce9b33c8cf24839a808aa1c3642d9ea1885a 100644 (file)
--- a/src/os/FileJournal.h
+++ b/src/os/FileJournal.h
@@ -287,6 +287,7 @@ private:
   // write thread
   Mutex write_lock;
   bool write_stop;
+  bool aio_stop;
 
   Cond commit_cond;
 
@@ -378,6 +379,7 @@ private:
     throttle_bytes(g_ceph_context, "filestore_bytes"),
     write_lock("FileJournal::write_lock", false, true, false, g_ceph_context),
     write_stop(false),
+    aio_stop(false),
     write_thread(this),
     write_finish_thread(this) { }
   ~FileJournal() {