From 972c7416deae2dd3a763643be6c9334d4edd1c17 Mon Sep 17 00:00:00 2001
From: Amir Vadai <amir@vadai.me>
Date: Wed, 12 Apr 2017 15:17:56 +0300
Subject: [PATCH] msg/async/rdma: Use RDMA resources only after fork

Thanks to previous patch [1], no need to access RDMA resources before
the fork. Initialize Infiniband class only before a connection is
established or a listener is created. [1] is making sure that the call
to RDMAWorker::listen() is postponed till after the fork.

[1] - 7393db45644d ("msg/async: Postpone bind if network stack is not ready")

Issue: 995322
Change-Id: I8ea246b2e03c8c9533bc324b2b8d142eb3d1ed4d
Signed-off-by: Amir Vadai <amir@vadai.me>
---
 src/msg/async/rdma/Infiniband.cc | 15 ++++++++++++++-
 src/msg/async/rdma/Infiniband.h  |  6 +++++-
 src/msg/async/rdma/RDMAStack.cc  | 27 ++++-----------------------
 src/msg/async/rdma/RDMAStack.h   |  4 +---
 4 files changed, 24 insertions(+), 28 deletions(-)

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index 1edb4ca71d674..a7bec09e4f324 100644
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -588,7 +588,7 @@ int Infiniband::MemoryManager::get_channel_buffers(std::vector<Chunk*> &chunks,
 
 
 Infiniband::Infiniband(CephContext *cct)
-  : device_list(new DeviceList(cct, this))
+  : cct(cct), lock("IB lock")
 {
 }
 
@@ -600,6 +600,19 @@ Infiniband::~Infiniband()
   delete device_list;
 }
 
+void Infiniband::init()
+{
+  Mutex::Locker l(lock);
+
+  if (initialized)
+    return;
+
+  device_list = new DeviceList(cct, this);
+  initialized = true;
+
+  dispatcher->polling_start();
+}
+
 void Infiniband::set_dispatcher(RDMADispatcher *d)
 {
   assert(!d ^ !dispatcher);
diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index 53850eda1185d..9c492612a4fdc 100644
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -147,12 +147,16 @@ class Infiniband {
   };
 
  private:
-  DeviceList *device_list;
+  CephContext *cct;
+  Mutex lock;
+  bool initialized = false;
+  DeviceList *device_list = nullptr;
   RDMADispatcher *dispatcher = nullptr;
 
  public:
   explicit Infiniband(CephContext *c);
   ~Infiniband();
+  void init();
 
   void set_dispatcher(RDMADispatcher *d);
 
diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc
index 66c7e5634abc9..88ce6bf168ebf 100644
--- a/src/msg/async/rdma/RDMAStack.cc
+++ b/src/msg/async/rdma/RDMAStack.cc
@@ -73,8 +73,6 @@ RDMADispatcher::RDMADispatcher(CephContext* c, RDMAStack* s)
 
   perf_logger = plb.create_perf_counters();
   cct->get_perfcounters_collection()->add(perf_logger);
-
-  cct->register_fork_watcher(this);
 }
 
 void RDMADispatcher::polling_start()
@@ -283,26 +281,6 @@ void RDMADispatcher::erase_qpn(uint32_t qpn)
   erase_qpn_lockless(qpn);
 }
 
-void RDMADispatcher::handle_pre_fork()
-{
-  polling_stop();
-  done = false;
-
-  global_infiniband->handle_pre_fork();
-
-  global_infiniband.destroy();
-}
-
-void RDMADispatcher::handle_post_fork()
-{
-  if (!global_infiniband) {
-    global_infiniband.construct(cct);
-    global_infiniband->set_dispatcher(this);
-  }
-
-  polling_start();
-}
-
 void RDMADispatcher::handle_tx_event(Device *ibdev, ibv_wc *cqe, int n)
 {
   std::vector<Chunk*> tx_chunks;
@@ -410,6 +388,8 @@ void RDMAWorker::initialize()
 
 int RDMAWorker::listen(entity_addr_t &sa, const SocketOptions &opt,ServerSocket *sock)
 {
+  global_infiniband->init();
+
   auto p = new RDMAServerConnTCP(cct, global_infiniband.get(), get_stack()->get_dispatcher(), this, sa);
   int r = p->listen(sa, opt);
   if (r < 0) {
@@ -423,6 +403,8 @@ int RDMAWorker::listen(entity_addr_t &sa, const SocketOptions &opt,ServerSocket
 
 int RDMAWorker::connect(const entity_addr_t &addr, const SocketOptions &opts, ConnectedSocket *socket)
 {
+  global_infiniband->init();
+
   RDMAConnectedSocketImpl* p = new RDMAConnectedSocketImpl(cct, global_infiniband.get(), get_stack()->get_dispatcher(), this);
   int r = p->try_connect(addr, opts);
 
@@ -507,7 +489,6 @@ RDMAStack::RDMAStack(CephContext *cct, const string &t): NetworkStack(cct, t)
   ldout(cct, 20) << __func__ << " constructing RDMAStack..." << dendl;
   dispatcher = new RDMADispatcher(cct, this);
   global_infiniband->set_dispatcher(dispatcher);
-  dispatcher->polling_start();
 
   unsigned num = get_num_worker();
   for (unsigned i = 0; i < num; ++i) {
diff --git a/src/msg/async/rdma/RDMAStack.h b/src/msg/async/rdma/RDMAStack.h
index 35eb531a1530c..8e91d42f533b0 100644
--- a/src/msg/async/rdma/RDMAStack.h
+++ b/src/msg/async/rdma/RDMAStack.h
@@ -62,7 +62,7 @@ enum {
 };
 
 
-class RDMADispatcher : public CephContext::ForkWatcher {
+class RDMADispatcher {
   typedef Infiniband::MemoryManager::Chunk Chunk;
   typedef Infiniband::QueuePair QueuePair;
 
@@ -125,8 +125,6 @@ class RDMADispatcher : public CephContext::ForkWatcher {
   void erase_qpn_lockless(uint32_t qpn);
   void erase_qpn(uint32_t qpn);
   void notify_pending_workers();
-  virtual void handle_pre_fork() override;
-  virtual void handle_post_fork() override;
   void handle_tx_event(Device *ibdev, ibv_wc *cqe, int n);
   void post_tx_buffer(Device *ibdev, std::vector<Chunk*> &chunks);
 
-- 
2.39.5