msg/async/rdma: make sure no alive connection when Dispatcher exit

Signed-off-by: Haomai Wang <haomai@xsky.com>

diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc
index a7aa7e03d3518716577d7e0434f3de4e9ce45a9f..e3e3c2037c43a28fd7a2d982b6458f7d9ee22a1c 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.cc
+++ b/src/msg/async/rdma/RDMAStack.cc
@@ -36,6 +36,7 @@ RDMADispatcher::~RDMADispatcher()
   ldout(cct, 20) << __func__ << " destructing rdma dispatcher" << dendl;
 
   assert(qp_conns.empty());
+  assert(num_qp_conn == 0);
   assert(dead_queue_pairs.empty());
   assert(num_dead_queue_pair == 0);
 
@@ -213,7 +214,7 @@ void RDMADispatcher::polling()
           --num_dead_queue_pair;
         }
       }
-      if (done)
+      if (!num_qp_conn && done)
         break;
 
       if ((ceph_clock_now() - last_inactive).to_nsec() / 1000 > cct->_conf->ms_async_rdma_polling_us) {
@@ -279,6 +280,7 @@ int RDMADispatcher::register_qp(QueuePair *qp, RDMAConnectedSocketImpl* csi)
   Mutex::Locker l(lock);
   assert(!qp_conns.count(qp->get_local_qp_number()));
   qp_conns[qp->get_local_qp_number()] = std::make_pair(qp, csi);
+  ++num_qp_conn;
   return fd;
 }
 
@@ -301,6 +303,7 @@ void RDMADispatcher::erase_qpn(uint32_t qpn)
   ++num_dead_queue_pair;
   dead_queue_pairs.push_back(it->second.first);
   qp_conns.erase(it);
+  --num_qp_conn;
 }
 
 void RDMADispatcher::handle_pre_fork()

diff --git a/src/msg/async/rdma/RDMAStack.h b/src/msg/async/rdma/RDMAStack.h
index 7afc378efc86b701cfe49c9efb6962f25891bbb0..0e855a3e73bdb110c8fa51783af16a8f2f160234 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.h
+++ b/src/msg/async/rdma/RDMAStack.h
@@ -73,6 +73,7 @@ class RDMADispatcher : public CephContext::ForkWatcher {
   EventCallbackRef async_handler;
   bool done = false;
   std::atomic<uint64_t> num_dead_queue_pair = {0};
+  std::atomic<uint64_t> num_qp_conn = {0};
   Mutex lock; // protect `qp_conns`, `dead_queue_pairs`
   // qp_num -> InfRcConnection
   // The main usage of `qp_conns` is looking up connection by qp_num,