msg/async/rdma: Fix crash when running traffic

Root cause- called lock mutex twice by same thread.
Solution- Added function erase_qpn_lockless() and update to call new function.

issue:1007313

Change-Id: Iacc6b217331381219e269682d3fae4061a1922f9
Signed-off-by: Sarit Zubakov <saritz@mellanox.com>

diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc
index e946afad0142d56d2ca4a3a2defdc044927a1bf3..86b6c1b4a71428486368ad2c3cf83cd4869dca45 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.cc
+++ b/src/msg/async/rdma/RDMAStack.cc
@@ -119,7 +119,7 @@ void RDMADispatcher::handle_async_event()
       } else {
         ldout(cct, 1) << __func__ << " it's not forwardly stopped by us, reenable=" << conn << dendl;
         conn->fault();
-        erase_qpn(qpn);
+        erase_qpn_lockless(qpn);
       }
     } else {
       ldout(cct, 1) << __func__ << " ibv_get_async_event: dev=" << global_infiniband->get_device()->ctxt
@@ -295,9 +295,8 @@ RDMAConnectedSocketImpl* RDMADispatcher::get_conn_lockless(uint32_t qp)
   return it->second.second;
 }
 
-void RDMADispatcher::erase_qpn(uint32_t qpn)
+void RDMADispatcher::erase_qpn_lockless(uint32_t qpn)
 {
-  Mutex::Locker l(lock);
   auto it = qp_conns.find(qpn);
   if (it == qp_conns.end())
     return ;
@@ -307,6 +306,12 @@ void RDMADispatcher::erase_qpn(uint32_t qpn)
   --num_qp_conn;
 }
 
+void RDMADispatcher::erase_qpn(uint32_t qpn)
+{
+  Mutex::Locker l(lock);
+  erase_qpn_lockless(qpn);
+}
+
 void RDMADispatcher::handle_pre_fork()
 {
   done = true;

diff --git a/src/msg/async/rdma/RDMAStack.h b/src/msg/async/rdma/RDMAStack.h
index 9e5d288bd3bd5f7f4cf733926d5e221676d3d9b4..e8a0b56ac8d297494308a9b6d5b0716d1af67b2e 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.h
+++ b/src/msg/async/rdma/RDMAStack.h
@@ -130,6 +130,7 @@ class RDMADispatcher : public CephContext::ForkWatcher {
   }
   RDMAStack* get_stack() { return stack; }
   RDMAConnectedSocketImpl* get_conn_lockless(uint32_t qp);
+  void erase_qpn_lockless(uint32_t qpn);
   void erase_qpn(uint32_t qpn);
   Infiniband::CompletionQueue* get_tx_cq() const { return tx_cq; }
   Infiniband::CompletionQueue* get_rx_cq() const { return rx_cq; }