From: Venky Shankar <vshankar@redhat.com>
Date: Thu, 28 Jan 2021 08:56:19 +0000 (-0500)
Subject: cephfs-mirror: gracefully shutdown threads, timers, etc..
X-Git-Tag: v16.2.0~158^2~3
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c369f221c72ad55022dd8fdac8e407bcebafc1e5;p=ceph.git

cephfs-mirror: gracefully shutdown threads, timers, etc..

Also, unregister signal handlers before exiting.

Signed-off-by: Venky Shankar <vshankar@redhat.com>
(cherry picked from commit a7de7f809b82de2f695b5328eb3cd39b65e64f65)
---

diff --git a/src/tools/cephfs_mirror/Mirror.cc b/src/tools/cephfs_mirror/Mirror.cc
index 62fdaa9f9ace..1cc0c766088f 100644
--- a/src/tools/cephfs_mirror/Mirror.cc
+++ b/src/tools/cephfs_mirror/Mirror.cc
@@ -35,11 +35,6 @@ public:
     : SafeTimer(cct, timer_lock, true) {
     init();
   }
-
-  ~SafeTimerSingleton() {
-    std::scoped_lock locker(timer_lock);
-    shutdown();
-  }
 };
 
 class ThreadPoolSingleton : public ThreadPool {
@@ -52,13 +47,6 @@ public:
 
     start();
   }
-
-  ~ThreadPoolSingleton() override {
-    work_queue->drain();
-    delete work_queue;
-
-    stop();
-  }
 };
 
 } // anonymous namespace
@@ -212,6 +200,7 @@ Mirror::Mirror(CephContext *cct, const std::vector<const char*> &args,
                          "cephfs::mirror::thread_pool", false, cct));
   auto safe_timer = &(cct->lookup_or_create_singleton_object<SafeTimerSingleton>(
                         "cephfs::mirror::safe_timer", false, cct));
+  m_thread_pool = thread_pool;
   m_work_queue = thread_pool->work_queue;
   m_timer = safe_timer;
   m_timer_lock = &safe_timer->timer_lock;
@@ -221,12 +210,25 @@ Mirror::Mirror(CephContext *cct, const std::vector<const char*> &args,
 
 Mirror::~Mirror() {
   dout(10) << dendl;
+  {
+    std::scoped_lock timer_lock(*m_timer_lock);
+    m_timer->shutdown();
+  }
+
+  m_work_queue->drain();
+  delete m_work_queue;
+  {
+    std::scoped_lock locker(m_lock);
+    m_thread_pool->stop();
+    m_cluster_watcher.reset();
+  }
 }
 
 int Mirror::init_mon_client() {
   dout(20) << dendl;
 
   m_monc->set_messenger(m_msgr);
+  m_msgr->add_dispatcher_head(m_monc);
   m_monc->set_want_keys(CEPH_ENTITY_TYPE_MON);
 
   int r = m_monc->init();
@@ -262,20 +264,14 @@ void Mirror::shutdown() {
   dout(20) << dendl;
 
   std::unique_lock locker(m_lock);
-  if (m_mirror_actions.empty()) {
-    return;
-  }
-
   m_stopping = true;
   m_cond.notify_all();
-  m_cond.wait(locker, [this] {return m_stopped;});
 }
 
 void Mirror::handle_signal(int signum) {
   dout(10) << ": signal=" << signum << dendl;
   ceph_assert(signum == SIGTERM || signum == SIGINT);
   shutdown();
-  ::exit(0);
 }
 
 void Mirror::handle_enable_mirroring(const Filesystem &filesystem,
@@ -537,9 +533,6 @@ void Mirror::run() {
       dout(10) << ": shutdown filesystem=" << filesystem << ", r=" << r << dendl;
     }
   }
-
-  m_stopped = true;
-  m_cond.notify_all();
 }
 
 } // namespace mirror
diff --git a/src/tools/cephfs_mirror/Mirror.h b/src/tools/cephfs_mirror/Mirror.h
index 77d6829c7dd2..933b04ac3467 100644
--- a/src/tools/cephfs_mirror/Mirror.h
+++ b/src/tools/cephfs_mirror/Mirror.h
@@ -9,6 +9,7 @@
 #include <vector>
 
 #include "common/ceph_mutex.h"
+#include "common/WorkQueue.h"
 #include "mds/FSMap.h"
 #include "ClusterWatcher.h"
 #include "FSMirror.h"
@@ -88,13 +89,13 @@ private:
   Messenger *m_msgr;
   ClusterListener m_listener;
 
+  ThreadPool *m_thread_pool = nullptr;
   ContextWQ *m_work_queue = nullptr;
   SafeTimer *m_timer = nullptr;
   ceph::mutex *m_timer_lock = nullptr;
   Context *m_timer_task = nullptr;
 
   bool m_stopping = false;
-  bool m_stopped = false;
   std::unique_ptr<ClusterWatcher> m_cluster_watcher;
   std::map<Filesystem, MirrorAction> m_mirror_actions;
 
diff --git a/src/tools/cephfs_mirror/main.cc b/src/tools/cephfs_mirror/main.cc
index d363159a0eb6..2301cb7ef48f 100644
--- a/src/tools/cephfs_mirror/main.cc
+++ b/src/tools/cephfs_mirror/main.cc
@@ -82,12 +82,18 @@ int main(int argc, const char **argv) {
   }
 
   mirror->run();
+  delete mirror;
 
 cleanup:
   monc.shutdown();
 cleanup_messenger:
   msgr->shutdown();
-  delete mirror;
+  msgr->wait();
+  delete msgr;
+
+  unregister_async_signal_handler(SIGINT, handle_signal);
+  unregister_async_signal_handler(SIGTERM, handle_signal);
+  shutdown_async_signal_handler();
 
   return r < 0 ? EXIT_SUCCESS : EXIT_FAILURE;
 }