xio: clean xcon on connection teardown

Accelio could raise connection teardown event without
a disconnected/closed/refused/error event before it which will
leave the xcon reference in XioMessenger entity map and conn list
which will also cause an assertion from boost.

Signed-off-by: Roi Dayan <roid@mellanox.com>

diff --git a/src/msg/xio/XioMessenger.cc b/src/msg/xio/XioMessenger.cc
index abff1eea71d2c1692b785e6cbc268b87604cf53d..5233ce5066dec1e0cc2cad27342668ab9838ab1b 100644 (file)
--- a/src/msg/xio/XioMessenger.cc
+++ b/src/msg/xio/XioMessenger.cc
@@ -534,22 +534,7 @@ int XioMessenger::session_event(struct xio_session *session,
     ldout(cct,2) << xio_session_event_str(event_data->event)
       << " xcon " << xcon << " session " << session  << dendl;
     if (likely(!!xcon)) {
-      Spinlock::Locker lckr(conns_sp);
-      XioConnection::EntitySet::iterator conn_iter =
-       conns_entity_map.find(xcon->peer, XioConnection::EntityComp());
-      if (conn_iter != conns_entity_map.end()) {
-       XioConnection *xcon2 = &(*conn_iter);
-       if (xcon == xcon2) {
-         conns_entity_map.erase(conn_iter);
-       }
-      }
-      /* check if citer on conn_list */
-      if (xcon->conns_hook.is_linked()) {
-        /* now find xcon on conns_list and erase */
-        XioConnection::ConnList::iterator citer =
-            XioConnection::ConnList::s_iterator_to(*xcon);
-        conns_list.erase(citer);
-      }
+      unregister_xcon(xcon);
       xcon->on_disconnect_event();
     }
     break;
@@ -557,6 +542,11 @@ int XioMessenger::session_event(struct xio_session *session,
     xcon = static_cast<XioConnection*>(event_data->conn_user_context);
     ldout(cct,2) << xio_session_event_str(event_data->event)
       << " xcon " << xcon << " session " << session << dendl;
+    /*
+     * There are flows where Accelio sends teardown event without going
+     * through disconnect event. so we make sure we cleaned the connection.
+     */
+    unregister_xcon(xcon);
     xcon->on_teardown_event();
     break;
   case XIO_SESSION_TEARDOWN_EVENT:
@@ -1043,6 +1033,28 @@ ConnectionRef XioMessenger::get_loopback_connection()
   return (loop_con.get());
 } /* get_loopback_connection */
 
+void XioMessenger::unregister_xcon(XioConnection *xcon)
+{
+  Spinlock::Locker lckr(conns_sp);
+
+  XioConnection::EntitySet::iterator conn_iter =
+       conns_entity_map.find(xcon->peer, XioConnection::EntityComp());
+  if (conn_iter != conns_entity_map.end()) {
+       XioConnection *xcon2 = &(*conn_iter);
+       if (xcon == xcon2) {
+         conns_entity_map.erase(conn_iter);
+       }
+  }
+
+  /* check if citer on conn_list */
+  if (xcon->conns_hook.is_linked()) {
+    /* now find xcon on conns_list and erase */
+    XioConnection::ConnList::iterator citer =
+        XioConnection::ConnList::s_iterator_to(*xcon);
+    conns_list.erase(citer);
+  }
+}
+
 void XioMessenger::mark_down(const entity_addr_t& addr)
 {
   entity_inst_t inst(entity_name_t(), addr);

diff --git a/src/msg/xio/XioMessenger.h b/src/msg/xio/XioMessenger.h
index 38390c408fdd7966b42da62294d70be0ffe83c87..56dae250aeb8871c65dc4a7ddd62a93337879c65 100644 (file)
--- a/src/msg/xio/XioMessenger.h
+++ b/src/msg/xio/XioMessenger.h
@@ -130,6 +130,7 @@ public:
 
   virtual ConnectionRef get_loopback_connection();
 
+  void unregister_xcon(XioConnection *xcon);
   virtual void mark_down(const entity_addr_t& a);
   virtual void mark_down(Connection *con);
   virtual void mark_down_all();