From: Haomai Wang <haomai@xsky.com>
Date: Thu, 9 Mar 2017 14:51:41 +0000 (+0800)
Subject: msg/async/rdma: make sure no alive connection when Dispatcher exit
X-Git-Tag: v12.0.1~70^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2dafbae9c23cc7d5fd707954ceb94f45b8c39959;p=ceph.git

msg/async/rdma: make sure no alive connection when Dispatcher exit

Signed-off-by: Haomai Wang <haomai@xsky.com>
---

diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc
index a7aa7e03d35..e3e3c2037c4 100644
--- a/src/msg/async/rdma/RDMAStack.cc
+++ b/src/msg/async/rdma/RDMAStack.cc
@@ -36,6 +36,7 @@ RDMADispatcher::~RDMADispatcher()
   ldout(cct, 20) << __func__ << " destructing rdma dispatcher" << dendl;
 
   assert(qp_conns.empty());
+  assert(num_qp_conn == 0);
   assert(dead_queue_pairs.empty());
   assert(num_dead_queue_pair == 0);
 
@@ -213,7 +214,7 @@ void RDMADispatcher::polling()
           --num_dead_queue_pair;
         }
       }
-      if (done)
+      if (!num_qp_conn && done)
         break;
 
       if ((ceph_clock_now() - last_inactive).to_nsec() / 1000 > cct->_conf->ms_async_rdma_polling_us) {
@@ -279,6 +280,7 @@ int RDMADispatcher::register_qp(QueuePair *qp, RDMAConnectedSocketImpl* csi)
   Mutex::Locker l(lock);
   assert(!qp_conns.count(qp->get_local_qp_number()));
   qp_conns[qp->get_local_qp_number()] = std::make_pair(qp, csi);
+  ++num_qp_conn;
   return fd;
 }
 
@@ -301,6 +303,7 @@ void RDMADispatcher::erase_qpn(uint32_t qpn)
   ++num_dead_queue_pair;
   dead_queue_pairs.push_back(it->second.first);
   qp_conns.erase(it);
+  --num_qp_conn;
 }
 
 void RDMADispatcher::handle_pre_fork()
diff --git a/src/msg/async/rdma/RDMAStack.h b/src/msg/async/rdma/RDMAStack.h
index 7afc378efc8..0e855a3e73b 100644
--- a/src/msg/async/rdma/RDMAStack.h
+++ b/src/msg/async/rdma/RDMAStack.h
@@ -73,6 +73,7 @@ class RDMADispatcher : public CephContext::ForkWatcher {
   EventCallbackRef async_handler;
   bool done = false;
   std::atomic<uint64_t> num_dead_queue_pair = {0};
+  std::atomic<uint64_t> num_qp_conn = {0};
   Mutex lock; // protect `qp_conns`, `dead_queue_pairs`
   // qp_num -> InfRcConnection
   // The main usage of `qp_conns` is looking up connection by qp_num,