svcrdma: Post Receives in the Receive completion handler

This change improves Receive efficiency by posting Receives only
on the same CPU that handles Receive completion. Improved latency
and throughput has been noted with this change.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: J. Bruce Fields <bfields@redhat.com>

diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index 995c6fe9ee907ff335d6be33ce68a5107abf06c4..4b731b046bcde187630d02b16ec19f33dd405afd 100644 (file)
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -185,8 +185,6 @@ extern void svc_rdma_wc_reg(struct ib_cq *, struct ib_wc *);
 extern void svc_rdma_wc_read(struct ib_cq *, struct ib_wc *);
 extern void svc_rdma_wc_inv(struct ib_cq *, struct ib_wc *);
 extern int svc_rdma_send(struct svcxprt_rdma *, struct ib_send_wr *);
-extern int svc_rdma_post_recv(struct svcxprt_rdma *, gfp_t);
-extern int svc_rdma_repost_recv(struct svcxprt_rdma *, gfp_t);
 extern int svc_rdma_create_listen(struct svc_serv *, int, struct sockaddr *);
 extern struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *);
 extern void svc_rdma_put_context(struct svc_rdma_op_ctxt *, int);

diff --git a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
index af7893501e40acdbaf678a373f721264cf398029..a73632ca9048228a6aa29b7f182a3ddd71049e79 100644 (file)
--- a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
@@ -95,7 +95,6 @@ out_shortreply:
 out_notfound:
        dprintk("svcrdma: unrecognized bc reply: xprt=%p, xid=%08x\n",
                xprt, be32_to_cpu(xid));
-
        goto out_unlock;
 }
 
@@ -129,10 +128,6 @@ static int svc_rdma_bc_sendto(struct svcxprt_rdma *rdma,
        if (ret < 0)
                goto out_err;
 
-       ret = svc_rdma_repost_recv(rdma, GFP_NOIO);
-       if (ret)
-               goto out_err;
-
        /* Bump page refcnt so Send completion doesn't release
         * the rq_buffer before all retransmits are complete.
         */

diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
index ad4bd62eebf1ac0ce765f68cab19b916080d7409..19e9c6b3304272e1f9e9f289d3726a1b2845e60b 100644 (file)
--- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
@@ -400,10 +400,6 @@ static void svc_rdma_send_error(struct svcxprt_rdma *xprt,
        struct page *page;
        int ret;
 
-       ret = svc_rdma_repost_recv(xprt, GFP_KERNEL);
-       if (ret)
-               return;
-
        page = alloc_page(GFP_KERNEL);
        if (!page)
                return;
@@ -554,8 +550,6 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp)
                ret = svc_rdma_handle_bc_reply(xprt->xpt_bc_xprt, p,
                                               &rqstp->rq_arg);
                svc_rdma_put_context(ctxt, 0);
-               if (ret)
-                       goto repost;
                return ret;
        }
 
@@ -590,6 +584,5 @@ out_postfail:
 
 out_drop:
        svc_rdma_put_context(ctxt, 1);
-repost:
-       return svc_rdma_repost_recv(rdma_xprt, GFP_KERNEL);
+       return 0;
 }

diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
index 7c3a211e0e9a9e17ed388696ebff789512289cf4..649441d5087d9abd0c3a2db1c488d4d3d3ef5c62 100644 (file)
--- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
@@ -674,9 +674,6 @@ int svc_rdma_sendto(struct svc_rqst *rqstp)
                svc_rdma_xdr_encode_reply_chunk(rdma_resp, rp_ch, ret);
        }
 
-       ret = svc_rdma_post_recv(rdma, GFP_KERNEL);
-       if (ret)
-               goto err1;
        ret = svc_rdma_send_reply_msg(rdma, rdma_argp, rdma_resp, rqstp,
                                      wr_lst, rp_ch);
        if (ret < 0)
@@ -687,9 +684,6 @@ int svc_rdma_sendto(struct svc_rqst *rqstp)
        if (ret != -E2BIG && ret != -EINVAL)
                goto err1;
 
-       ret = svc_rdma_post_recv(rdma, GFP_KERNEL);
-       if (ret)
-               goto err1;
        ret = svc_rdma_send_error_msg(rdma, rdma_resp, rqstp);
        if (ret < 0)
                goto err0;

diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index 46ec069150d50ff53e93a7f17b0d716fa80503a2..9ad12a215b514b146f6d9234f5e24be308099b05 100644 (file)
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -58,6 +58,7 @@
 
 #define RPCDBG_FACILITY        RPCDBG_SVCXPRT
 
+static int svc_rdma_post_recv(struct svcxprt_rdma *xprt);
 static struct svcxprt_rdma *rdma_create_xprt(struct svc_serv *, int);
 static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
                                        struct net *net,
@@ -320,6 +321,8 @@ static void svc_rdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc)
        list_add_tail(&ctxt->list, &xprt->sc_rq_dto_q);
        spin_unlock(&xprt->sc_rq_dto_lock);
 
+       svc_rdma_post_recv(xprt);
+
        set_bit(XPT_DATA, &xprt->sc_xprt.xpt_flags);
        if (test_bit(RDMAXPRT_CONN_PENDING, &xprt->sc_flags))
                goto out;
@@ -404,7 +407,8 @@ static struct svcxprt_rdma *rdma_create_xprt(struct svc_serv *serv,
        return cma_xprt;
 }
 
-int svc_rdma_post_recv(struct svcxprt_rdma *xprt, gfp_t flags)
+static int
+svc_rdma_post_recv(struct svcxprt_rdma *xprt)
 {
        struct ib_recv_wr recv_wr, *bad_recv_wr;
        struct svc_rdma_op_ctxt *ctxt;
@@ -423,7 +427,7 @@ int svc_rdma_post_recv(struct svcxprt_rdma *xprt, gfp_t flags)
                        pr_err("svcrdma: Too many sges (%d)\n", sge_no);
                        goto err_put_ctxt;
                }
-               page = alloc_page(flags);
+               page = alloc_page(GFP_KERNEL);
                if (!page)
                        goto err_put_ctxt;
                ctxt->pages[sge_no] = page;
@@ -459,21 +463,6 @@ int svc_rdma_post_recv(struct svcxprt_rdma *xprt, gfp_t flags)
        return -ENOMEM;
 }
 
-int svc_rdma_repost_recv(struct svcxprt_rdma *xprt, gfp_t flags)
-{
-       int ret = 0;
-
-       ret = svc_rdma_post_recv(xprt, flags);
-       if (ret) {
-               pr_err("svcrdma: could not post a receive buffer, err=%d.\n",
-                      ret);
-               pr_err("svcrdma: closing transport %p.\n", xprt);
-               set_bit(XPT_CLOSE, &xprt->sc_xprt.xpt_flags);
-               ret = -ENOTCONN;
-       }
-       return ret;
-}
-
 static void
 svc_rdma_parse_connect_private(struct svcxprt_rdma *newxprt,
                               struct rdma_conn_param *param)
@@ -833,7 +822,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
 
        /* Post receive buffers */
        for (i = 0; i < newxprt->sc_max_requests; i++) {
-               ret = svc_rdma_post_recv(newxprt, GFP_KERNEL);
+               ret = svc_rdma_post_recv(newxprt);
                if (ret) {
                        dprintk("svcrdma: failure posting receive buffers\n");
                        goto errout;