XIO: Handle queued incoming XIO messages during retry

1. Move queued incoming XIO messages to be released into the requeue Q
for retry once resources free up
2. During connection close, freeup queued incoming XIO messages to be
released
3. Handle a case where xio_send_msg() returns -1. In cases where
xio_errno() is also -1, XIO would have queued the message(s) to the
connection queue.

Signed-off-by: Raju Kurunkad <raju.kurunkad@sandisk.com>

diff --git a/src/msg/xio/XioConnection.cc b/src/msg/xio/XioConnection.cc
index 1738a663292e24a0d35bd4de98bcb4a284c8116a..3b63320a5c048387d5dca863f819e24bf67d722c 100644 (file)
--- a/src/msg/xio/XioConnection.cc
+++ b/src/msg/xio/XioConnection.cc
@@ -539,12 +539,23 @@ int XioConnection::discard_input_queue(uint32_t flags)
   for (ix = 0; ix < q_size; ++ix) {
     XioSubmit::Queue::iterator q_iter = deferred_q.begin();
     XioSubmit* xs = &(*q_iter);
-    assert(xs->type == XioSubmit::OUTGOING_MSG);
-    XioMsg* xmsg = static_cast<XioMsg*>(xs);
-    deferred_q.erase(q_iter);
-    // release once for each chained xio_msg
-    for (ix = 0; ix < int(xmsg->hdr.msg_cnt); ++ix)
-      xmsg->put();
+    XioMsg* xmsg;
+    switch (xs->type) {
+      case XioSubmit::OUTGOING_MSG:
+       xmsg = static_cast<XioMsg*>(xs);
+       deferred_q.erase(q_iter);
+       // release once for each chained xio_msg
+       for (ix = 0; ix < int(xmsg->hdr.msg_cnt); ++ix)
+         xmsg->put();
+       break;
+      case XioSubmit::INCOMING_MSG_RELEASE:
+       deferred_q.erase(q_iter);
+       portal->release_xio_rsp(static_cast<XioRsp*>(xs));
+       break;
+      default:
+       ldout(msgr->cct,0) << __func__ << ": Unknown Msg type " << xs->type << dendl;
+       break;
+    }
   }
 
   return 0;

diff --git a/src/msg/xio/XioPortal.h b/src/msg/xio/XioPortal.h
index ea644a9f987e1ead6e5681bc1fd71247b9b644f1..9d98548038c2d991b75247b345ee1c0f008250fa 100644 (file)
--- a/src/msg/xio/XioPortal.h
+++ b/src/msg/xio/XioPortal.h
@@ -213,9 +213,10 @@ public:
     while (q_iter != send_q.end()) {
       XioSubmit *xs = &(*q_iter);
       // skip retires and anything for other connections
-      if ((xs->type != XioSubmit::OUTGOING_MSG) ||
-         (xs->xcon != xcon))
+      if (xs->xcon != xcon) {
+       q_iter++;
        continue;
+      }
       xmsg = static_cast<XioMsg*>(xs);
       q_iter = send_q.erase(q_iter);
       requeue_q.push_back(*xmsg);
@@ -283,8 +284,19 @@ public:
                  print_ceph_msg(msgr->cct, "xio_send_msg", xmsg->m);
                }
                /* get the right Accelio's errno code */
-               if (unlikely(code))
-                 code = xio_errno();
+               if (unlikely(code)) {
+                 if ((code == -1) && (xio_errno() == -1)) {
+                   /* In case XIO does not have any credits to send,
+                    * it would still queue up the message(s) for transmission,
+                    * but would return -1 and errno would also be set to -1.
+                    * This needs to be treated as a success.
+                    */
+                   code = 0;
+                 }
+                 else {
+                   code = xio_errno();
+                 }
+               }
              } /* !ENOTCONN */
              if (unlikely(code)) {
                switch (code) {