From 2dafbae9c23cc7d5fd707954ceb94f45b8c39959 Mon Sep 17 00:00:00 2001 From: Haomai Wang Date: Thu, 9 Mar 2017 22:51:41 +0800 Subject: [PATCH] msg/async/rdma: make sure no alive connection when Dispatcher exit Signed-off-by: Haomai Wang --- src/msg/async/rdma/RDMAStack.cc | 5 ++++- src/msg/async/rdma/RDMAStack.h | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc index a7aa7e03d351..e3e3c2037c43 100644 --- a/src/msg/async/rdma/RDMAStack.cc +++ b/src/msg/async/rdma/RDMAStack.cc @@ -36,6 +36,7 @@ RDMADispatcher::~RDMADispatcher() ldout(cct, 20) << __func__ << " destructing rdma dispatcher" << dendl; assert(qp_conns.empty()); + assert(num_qp_conn == 0); assert(dead_queue_pairs.empty()); assert(num_dead_queue_pair == 0); @@ -213,7 +214,7 @@ void RDMADispatcher::polling() --num_dead_queue_pair; } } - if (done) + if (!num_qp_conn && done) break; if ((ceph_clock_now() - last_inactive).to_nsec() / 1000 > cct->_conf->ms_async_rdma_polling_us) { @@ -279,6 +280,7 @@ int RDMADispatcher::register_qp(QueuePair *qp, RDMAConnectedSocketImpl* csi) Mutex::Locker l(lock); assert(!qp_conns.count(qp->get_local_qp_number())); qp_conns[qp->get_local_qp_number()] = std::make_pair(qp, csi); + ++num_qp_conn; return fd; } @@ -301,6 +303,7 @@ void RDMADispatcher::erase_qpn(uint32_t qpn) ++num_dead_queue_pair; dead_queue_pairs.push_back(it->second.first); qp_conns.erase(it); + --num_qp_conn; } void RDMADispatcher::handle_pre_fork() diff --git a/src/msg/async/rdma/RDMAStack.h b/src/msg/async/rdma/RDMAStack.h index 7afc378efc86..0e855a3e73bd 100644 --- a/src/msg/async/rdma/RDMAStack.h +++ b/src/msg/async/rdma/RDMAStack.h @@ -73,6 +73,7 @@ class RDMADispatcher : public CephContext::ForkWatcher { EventCallbackRef async_handler; bool done = false; std::atomic num_dead_queue_pair = {0}; + std::atomic num_qp_conn = {0}; Mutex lock; // protect `qp_conns`, `dead_queue_pairs` // qp_num -> InfRcConnection // The main usage of `qp_conns` is looking up connection by qp_num, -- 2.47.3