msg/async/rdma: Make poll_blocking() poll for async events in addition to tx/rx

Issue: 995322
Change-Id: I62e75f4c4e574ba97e24b20a4d7b94c56c605bb5
Signed-off-by: Amir Vadai <amir@vadai.me>

diff --git a/src/msg/async/rdma/Device.cc b/src/msg/async/rdma/Device.cc
index 392ed7f69c75b882e289ea142d24a6350c45f63b..ee5bbd73fd884cf9c3c1d4aada97f94dbf8466da 100644 (file)
--- a/src/msg/async/rdma/Device.cc
+++ b/src/msg/async/rdma/Device.cc
@@ -411,10 +411,10 @@ DeviceList::DeviceList(CephContext *cct, Infiniband *ib)
   }
   devices = new Device*[num];
 
-  poll_fds = new struct pollfd[2 * num];
+  poll_fds = new struct pollfd[3 * num];
 
   for (int i = 0; i < num; ++i) {
-    struct pollfd *pfd = &poll_fds[i * 2];
+    struct pollfd *pfd = &poll_fds[i * 3];
     struct Device *d;
 
     d = new Device(cct, ib, device_list[i]);
@@ -427,6 +427,10 @@ DeviceList::DeviceList(CephContext *cct, Infiniband *ib)
     pfd[1].fd = d->rx_cc->get_fd();
     pfd[1].events = POLLIN | POLLERR | POLLNVAL | POLLHUP;
     pfd[1].revents = 0;
+
+    pfd[2].fd = d->ctxt->async_fd;
+    pfd[2].events = POLLIN | POLLERR | POLLNVAL | POLLHUP;
+    pfd[2].revents = 0;
   }
 }
 
@@ -486,7 +490,7 @@ int DeviceList::poll_blocking(bool &done)
 {
   int r = 0;
   while (!done && r == 0) {
-    r = poll(poll_fds, num * 2, 100);
+    r = poll(poll_fds, num * 3, 100);
     if (r < 0) {
       r = -errno;
       lderr(cct) << __func__ << " poll failed " << r << dendl;
@@ -505,6 +509,8 @@ int DeviceList::poll_blocking(bool &done)
 
     if (d->rx_cc->get_cq_event())
       ldout(cct, 20) << __func__ << " " << *d << ": got rx cq event" << dendl;
+
+    d->handle_async_event();
   }
 
   return r;

diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc
index b4e7fab3483abdee7130cf9945a39cb88e25bae0..443ac7fa44885772d2526f5f6c3da0a6d02123fa 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.cc
+++ b/src/msg/async/rdma/RDMAStack.cc
@@ -207,7 +207,6 @@ void RDMADispatcher::polling()
         break;
 
       if ((ceph_clock_now() - last_inactive).to_nsec() / 1000 > cct->_conf->ms_async_rdma_polling_us) {
-        global_infiniband->handle_async_event();
         if (!rearmed) {
           // Clean up cq events after rearm notify ensure no new incoming event
           // arrived between polling and rearm