From bf24317bcdab4a99b9fdabef7e84fa99fdbc6720 Mon Sep 17 00:00:00 2001
From: Haomai Wang <haomaiwang@gmail.com>
Date: Thu, 26 Dec 2013 11:20:52 +0800
Subject: [PATCH] Fix WBThrottle thread disappear problem

New ceph_osd.cc code did ObjectStore init work before global_init_daemonize(),
and WBThrottle thread is created when objectstore constructed. So after
daemon(), WBThrottle thread won't exist in new process. It will result in
deadlock.

When "cur_ios" which is member of WBThrottle hits hard limit, there exists two
ways to decrease "cur_ios". The first is WBThrottle thread which is dead if
deamonize, another is SyncThread. SyncThread will block at op_tp.pause()
because thread in op_tp(threadpool) block at
wbthrottle.throttle(FileStore::doop). So no thread will continue process jobs
in filestore layer and all threads is waiting.

Fix #7062 (http://tracker.ceph.com/issues/7062)

Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
---
 src/os/FileStore.cc  |  4 ++++
 src/os/WBThrottle.cc | 23 ++++++++++++++++++-----
 src/os/WBThrottle.h  |  1 +
 3 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index 6a6c668eb00c5..22db6002db556 100644
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -1403,6 +1403,7 @@ int FileStore::mount()
     }
   }
 
+  wbthrottle.start();
   sync_thread.create();
 
   ret = journal_replay(initial_op_seq);
@@ -1420,6 +1421,8 @@ int FileStore::mount()
     lock.Unlock();
     sync_thread.join();
 
+    wbthrottle.stop();
+
     goto close_current_fd;
   }
 
@@ -1469,6 +1472,7 @@ int FileStore::umount()
   sync_cond.Signal();
   lock.Unlock();
   sync_thread.join();
+  wbthrottle.stop();
   op_tp.stop();
 
   journal_stop();
diff --git a/src/os/WBThrottle.cc b/src/os/WBThrottle.cc
index c5fb49505c55f..0354ceb8ccefc 100644
--- a/src/os/WBThrottle.cc
+++ b/src/os/WBThrottle.cc
@@ -10,7 +10,7 @@ WBThrottle::WBThrottle(CephContext *cct) :
   cur_ios(0), cur_size(0),
   cct(cct),
   logger(NULL),
-  stopping(false),
+  stopping(true),
   lock("WBThrottle::lock", false, true, false, cct),
   fs(XFS)
 {
@@ -34,20 +34,33 @@ WBThrottle::WBThrottle(CephContext *cct) :
     logger->set(i, 0);
 
   cct->_conf->add_observer(this);
-  create();
 }
 
 WBThrottle::~WBThrottle() {
   assert(cct);
+  cct->get_perfcounters_collection()->remove(logger);
+  delete logger;
+  cct->_conf->remove_observer(this);
+}
+
+void WBThrottle::start()
+{
+  {
+    Mutex::Locker l(lock);
+    stopping = false;
+  }
+  create();
+}
+
+void WBThrottle::stop()
+{
   {
     Mutex::Locker l(lock);
     stopping = true;
     cond.Signal();
   }
+
   join();
-  cct->get_perfcounters_collection()->remove(logger);
-  delete logger;
-  cct->_conf->remove_observer(this);
 }
 
 const char** WBThrottle::get_tracked_conf_keys() const
diff --git a/src/os/WBThrottle.h b/src/os/WBThrottle.h
index e418cf98d2a7f..b78123267508e 100644
--- a/src/os/WBThrottle.h
+++ b/src/os/WBThrottle.h
@@ -134,6 +134,7 @@ public:
   WBThrottle(CephContext *cct);
   ~WBThrottle();
 
+  void stop();
   /// Set fs as XFS or BTRFS
   void set_fs(FS new_fs) {
     Mutex::Locker l(lock);
-- 
2.39.5