Revert "msg/async/rdma: Use RDMA resources only after fork"

This reverts commit 972c7416deae2dd3a763643be6c9334d4edd1c17.

Change-Id: I4a5627ce02ef8e055c2088db41ac4897dd9a3425
Issue: 995322
Signed-off-by: Amir Vadai <amir@vadai.me>

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index b4a644e67d813755e3d15f1b0f9e1e54edbbe826..cda9d2bd206d120b1c40737f1c1be5e543127c3d 100644 (file)
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -585,7 +585,7 @@ int Infiniband::MemoryManager::get_channel_buffers(std::vector<Chunk*> &chunks,
 
 
 Infiniband::Infiniband(CephContext *cct)
-  : cct(cct), lock("IB lock")
+  : device_list(new DeviceList(cct, this))
 {
 }
 
@@ -597,19 +597,6 @@ Infiniband::~Infiniband()
   delete device_list;
 }
 
-void Infiniband::init()
-{
-  Mutex::Locker l(lock);
-
-  if (initialized)
-    return;
-
-  device_list = new DeviceList(cct, this);
-  initialized = true;
-
-  dispatcher->polling_start();
-}
-
 void Infiniband::set_dispatcher(RDMADispatcher *d)
 {
   assert(!d ^ !dispatcher);

diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index 7394ca92bad78c468a1eefab0abebcbfd0c576f5..58cf13e1938a5b228f8dbd68f349b3f2ff0a2d1c 100644 (file)
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -147,16 +147,12 @@ class Infiniband {
   };
 
  private:
-  CephContext *cct;
-  Mutex lock;
-  bool initialized = false;
-  DeviceList *device_list = nullptr;
+  DeviceList *device_list;
   RDMADispatcher *dispatcher = nullptr;
 
  public:
   explicit Infiniband(CephContext *c);
   ~Infiniband();
-  void init();
 
   void set_dispatcher(RDMADispatcher *d);
 

diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc
index b1dae3bcf5c2f02fa46d8ba9e4efc974c77e3391..3413f34015739dc0e4fb70a31faadb6096dbb1d0 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.cc
+++ b/src/msg/async/rdma/RDMAStack.cc
@@ -74,6 +74,8 @@ RDMADispatcher::RDMADispatcher(CephContext* c, RDMAStack* s)
 
   perf_logger = plb.create_perf_counters();
   cct->get_perfcounters_collection()->add(perf_logger);
+
+  cct->register_fork_watcher(this);
 }
 
 void RDMADispatcher::polling_start()
@@ -284,6 +286,26 @@ void RDMADispatcher::erase_qpn(uint32_t qpn)
   erase_qpn_lockless(qpn);
 }
 
+void RDMADispatcher::handle_pre_fork()
+{
+  polling_stop();
+  done = false;
+
+  global_infiniband->handle_pre_fork();
+
+  global_infiniband.destroy();
+}
+
+void RDMADispatcher::handle_post_fork()
+{
+  if (!global_infiniband) {
+    global_infiniband.construct(cct);
+    global_infiniband->set_dispatcher(this);
+  }
+
+  polling_start();
+}
+
 void RDMADispatcher::handle_tx_event(Device *ibdev, ibv_wc *cqe, int n)
 {
   std::vector<Chunk*> tx_chunks;
@@ -391,8 +413,6 @@ void RDMAWorker::initialize()
 
 int RDMAWorker::listen(entity_addr_t &sa, const SocketOptions &opt,ServerSocket *sock)
 {
-  global_infiniband->init();
-
   auto p = new RDMAServerConnTCP(cct, global_infiniband.get(), get_stack()->get_dispatcher(), this, sa);
   int r = p->listen(sa, opt);
   if (r < 0) {
@@ -406,8 +426,6 @@ int RDMAWorker::listen(entity_addr_t &sa, const SocketOptions &opt,ServerSocket
 
 int RDMAWorker::connect(const entity_addr_t &addr, const SocketOptions &opts, ConnectedSocket *socket)
 {
-  global_infiniband->init();
-
   RDMAConnectedSocketImpl* p = new RDMAConnectedSocketImpl(cct, global_infiniband.get(), get_stack()->get_dispatcher(), this);
   int r = p->try_connect(addr, opts);
 
@@ -492,6 +510,7 @@ RDMAStack::RDMAStack(CephContext *cct, const string &t): NetworkStack(cct, t)
   ldout(cct, 20) << __func__ << " constructing RDMAStack..." << dendl;
   dispatcher = new RDMADispatcher(cct, this);
   global_infiniband->set_dispatcher(dispatcher);
+  dispatcher->polling_start();
 
   unsigned num = get_num_worker();
   for (unsigned i = 0; i < num; ++i) {

diff --git a/src/msg/async/rdma/RDMAStack.h b/src/msg/async/rdma/RDMAStack.h
index ef1188b802375e3538ff7b7b0e3fdfdaaa26ee7d..9043ed1db7b7b2420efdba826f61e1be48971f6b 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.h
+++ b/src/msg/async/rdma/RDMAStack.h
@@ -63,7 +63,7 @@ enum {
 };
 
 
-class RDMADispatcher {
+class RDMADispatcher : public CephContext::ForkWatcher {
   typedef Infiniband::MemoryManager::Chunk Chunk;
   typedef Infiniband::QueuePair QueuePair;
 
@@ -126,6 +126,8 @@ class RDMADispatcher {
   void erase_qpn_lockless(uint32_t qpn);
   void erase_qpn(uint32_t qpn);
   void notify_pending_workers();
+  virtual void handle_pre_fork() override;
+  virtual void handle_post_fork() override;
   void handle_tx_event(Device *ibdev, ibv_wc *cqe, int n);
   void post_tx_buffer(Device *ibdev, std::vector<Chunk*> &chunks);