common/ceph_context: mark some singletones as drop-on-fork

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/TracepointProvider.h b/src/common/TracepointProvider.h
index 82a0359484ee406d902dadf0e899e168b2eba95f..a2b5f89c40e3cf293e8e4f4dd81646cecc141946 100644 (file)
--- a/src/common/TracepointProvider.h
+++ b/src/common/TracepointProvider.h
@@ -58,7 +58,7 @@ public:
   static void initialize(CephContext *cct) {
 #ifdef WITH_LTTNG
      cct->lookup_or_create_singleton_object<TypedSingleton<traits>>(
-       traits.library, cct);
+       traits.library, false, cct);
 #endif
   }
 

diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc
index 58623767800933dce0472e67e139448e149e0f58..65a30167fc5119bf650287104ea24dccb623cb65 100644 (file)
--- a/src/common/ceph_context.cc
+++ b/src/common/ceph_context.cc
@@ -644,7 +644,7 @@ CephContext::CephContext(uint32_t module_type_,
   _crypto_aes = CryptoHandler::create(CEPH_CRYPTO_AES);
   _crypto_random.reset(new CryptoRandom());
 
-  lookup_or_create_singleton_object<MempoolObs>("mempool_obs", this);
+  lookup_or_create_singleton_object<MempoolObs>("mempool_obs", false, this);
 }
 
 CephContext::~CephContext()
@@ -851,3 +851,33 @@ CryptoHandler *CephContext::get_crypto_handler(int type)
     return NULL;
   }
 }
+
+void CephContext::notify_pre_fork()
+{
+  {
+    std::lock_guard<ceph::spinlock> lg(_fork_watchers_lock);
+    for (auto &&t : _fork_watchers) {
+      t->handle_pre_fork();
+    }
+  }
+  {
+    // note: we don't hold a lock here, but we assume we are idle at
+    // fork time, which happens during process init and startup.
+    auto i = associated_objs.begin();
+    while (i != associated_objs.end()) {
+      if (associated_objs_drop_on_fork.count(i->first.first)) {
+       i = associated_objs.erase(i);
+      } else {
+       ++i;
+      }
+    }
+    associated_objs_drop_on_fork.clear();
+  }
+}
+
+void CephContext::notify_post_fork()
+{
+  ceph::spin_unlock(&_fork_watchers_lock);
+  for (auto &&t : _fork_watchers)
+    t->handle_post_fork();
+}

diff --git a/src/common/ceph_context.h b/src/common/ceph_context.h
index b06f6e067e04de629b1d686dcd716112c593fd41..55c59eca22e0568cfc6ef266cc6d58da6cf23d08 100644 (file)
--- a/src/common/ceph_context.h
+++ b/src/common/ceph_context.h
@@ -143,6 +143,7 @@ public:
 
   template<typename T, typename... Args>
   T& lookup_or_create_singleton_object(std::string_view name,
+                                      bool drop_on_fork,
                                       Args&&... args) {
     static_assert(sizeof(T) <= largest_singleton,
                  "Please increase largest singleton.");
@@ -151,6 +152,9 @@ public:
 
     auto i = associated_objs.find(std::make_pair(name, type));
     if (i == associated_objs.cend()) {
+      if (drop_on_fork) {
+       associated_objs_drop_on_fork.insert(std::string(name));
+      }
       i = associated_objs.emplace_hint(
        i,
        std::piecewise_construct,
@@ -211,17 +215,8 @@ public:
     _fork_watchers.push_back(w);
   }
 
-  void notify_pre_fork() {
-    std::lock_guard<ceph::spinlock> lg(_fork_watchers_lock);
-    for (auto &&t : _fork_watchers)
-      t->handle_pre_fork();
-  }
-
-  void notify_post_fork() {
-    ceph::spin_unlock(&_fork_watchers_lock);
-    for (auto &&t : _fork_watchers)
-      t->handle_post_fork();
-  }
+  void notify_pre_fork();
+  void notify_post_fork();
 
 private:
 
@@ -277,6 +272,7 @@ private:
   std::map<std::pair<std::string, std::type_index>,
           ceph::immobile_any<largest_singleton>,
           associated_objs_cmp> associated_objs;
+  std::set<std::string> associated_objs_drop_on_fork;
 
   ceph::spinlock _fork_watchers_lock;
   std::vector<ForkWatcher*> _fork_watchers;

diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index 9f8939b39728a320b3ab10221d41903dae5d6a69..05133636090e76fef6d0221e11689fdb0a4ffaea 100644 (file)
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -1178,7 +1178,7 @@ struct C_InvalidateCache : public Context {
                                           ContextWQ **op_work_queue) {
     auto thread_pool_singleton =
       &cct->lookup_or_create_singleton_object<ThreadPoolSingleton>(
-       "librbd::thread_pool", cct);
+       "librbd::thread_pool", false, cct);
     *thread_pool = thread_pool_singleton;
     *op_work_queue = thread_pool_singleton->op_work_queue;
   }
@@ -1187,7 +1187,7 @@ struct C_InvalidateCache : public Context {
                                     Mutex **timer_lock) {
     auto safe_timer_singleton =
       &cct->lookup_or_create_singleton_object<SafeTimerSingleton>(
-       "librbd::journal::safe_timer", cct);
+       "librbd::journal::safe_timer", false, cct);
     *timer = safe_timer_singleton;
     *timer_lock = &safe_timer_singleton->lock;
   }

diff --git a/src/librbd/ImageState.cc b/src/librbd/ImageState.cc
index 1fcb54a5342dd10e02226a1cfc00545d0c51d592..9ac9f1570a26be1445d1669163d1ea4db29c2ec5 100644 (file)
--- a/src/librbd/ImageState.cc
+++ b/src/librbd/ImageState.cc
@@ -213,7 +213,8 @@ private:
       return;
     }
     auto& thread_pool = m_cct->lookup_or_create_singleton_object<
-      ThreadPoolSingleton>("librbd::ImageUpdateWatchers::thread_pool", m_cct);
+      ThreadPoolSingleton>("librbd::ImageUpdateWatchers::thread_pool",
+                          false, m_cct);
     m_work_queue = new ContextWQ("librbd::ImageUpdateWatchers::op_work_queue",
                                 m_cct->_conf->get_val<int64_t>("rbd_op_thread_timeout"),
                                 &thread_pool);

diff --git a/src/librbd/Journal.cc b/src/librbd/Journal.cc
index 21eea0ef6eaa3838c9160e6ca912e13d1b3b5b31..4a13c2a0f1f8497765950be47704d819cf49bbb7 100644 (file)
--- a/src/librbd/Journal.cc
+++ b/src/librbd/Journal.cc
@@ -336,7 +336,7 @@ Journal<I>::Journal(I &image_ctx)
 
   auto thread_pool_singleton =
     &cct->lookup_or_create_singleton_object<ThreadPoolSingleton>(
-      "librbd::journal::thread_pool", cct);
+      "librbd::journal::thread_pool", false, cct);
   m_work_queue = new ContextWQ("librbd::journal::work_queue",
                                cct->_conf->get_val<int64_t>("rbd_op_thread_timeout"),
                                thread_pool_singleton);

diff --git a/src/librbd/TaskFinisher.h b/src/librbd/TaskFinisher.h
index 6ca0e19a6a0deb4d3d3c8a84cfc7870ca5c3cd89..8b9c94b56adf626a790ca6e6df36e13361f25df7 100644 (file)
--- a/src/librbd/TaskFinisher.h
+++ b/src/librbd/TaskFinisher.h
@@ -46,7 +46,7 @@ public:
   TaskFinisher(CephContext &cct) : m_cct(cct) {
     auto& singleton =
       cct.lookup_or_create_singleton_object<TaskFinisherSingleton>(
-       "librbd::TaskFinisher::m_safe_timer", &cct);
+       "librbd::TaskFinisher::m_safe_timer", false, &cct);
     m_lock = &singleton.m_lock;
     m_safe_timer = singleton.m_safe_timer;
     m_finisher = singleton.m_finisher;

diff --git a/src/msg/async/AsyncMessenger.cc b/src/msg/async/AsyncMessenger.cc
index d42220595738d0a7256390968d1430a7477dba9f..d56ca7bd6be0fbeae377517f8e945869b2b5202a 100644 (file)
--- a/src/msg/async/AsyncMessenger.cc
+++ b/src/msg/async/AsyncMessenger.cc
@@ -260,7 +260,7 @@ AsyncMessenger::AsyncMessenger(CephContext *cct, entity_name_t name,
     transport_type = "dpdk";
 
   auto single = &cct->lookup_or_create_singleton_object<StackSingleton>(
-    "AsyncMessenger::NetworkStack::" + transport_type, cct);
+    "AsyncMessenger::NetworkStack::" + transport_type, true, cct);
   single->ready(transport_type);
   stack = single->stack.get();
   stack->start();

diff --git a/src/msg/async/Event.cc b/src/msg/async/Event.cc
index eadfac4f75554b18380914f959eec386cae0fb5d..9038df29c6973d1e295876f7f99adb4aa454b2d0 100644 (file)
--- a/src/msg/async/Event.cc
+++ b/src/msg/async/Event.cc
@@ -191,7 +191,7 @@ void EventCenter::set_owner()
   if (!global_centers) {
     global_centers = &cct->lookup_or_create_singleton_object<
       EventCenter::AssociatedCenters>(
-       "AsyncMessenger::EventCenter::global_center::" + type);
+       "AsyncMessenger::EventCenter::global_center::" + type, true);
     assert(global_centers);
     global_centers->centers[idx] = this;
     if (driver->need_wakeup()) {

diff --git a/src/tools/rbd_mirror/Mirror.cc b/src/tools/rbd_mirror/Mirror.cc
index 1989c78ad0672ca9f952375b634d9269563355e1..3483d662da1e5dcfc62d560ae607603edc837503 100644 (file)
--- a/src/tools/rbd_mirror/Mirror.cc
+++ b/src/tools/rbd_mirror/Mirror.cc
@@ -206,7 +206,7 @@ Mirror::Mirror(CephContext *cct, const std::vector<const char*> &args) :
 {
   m_threads =
     &(cct->lookup_or_create_singleton_object<Threads<librbd::ImageCtx>>(
-       "rbd_mirror::threads", cct));
+       "rbd_mirror::threads", false, cct));
   m_service_daemon.reset(new ServiceDaemon<>(m_cct, m_local, m_threads));
 }