From 37ec0e14f15108c86840da5d9addefb61c2da399 Mon Sep 17 00:00:00 2001 From: Adir Lev Date: Sun, 4 Dec 2016 11:30:12 +0000 Subject: [PATCH] msg/async/rdma: set correct value to memory manager set the correct max_send_wr and max_recv_wr print log messages accordingly Change-Id: I97a562bc994f14b87724fcf84baca4cdbf0f8e0d Signed-off-by: Adir Lev --- src/msg/async/rdma/Infiniband.cc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc index 968ff9d90d1e4..14a13cb0602f8 100644 --- a/src/msg/async/rdma/Infiniband.cc +++ b/src/msg/async/rdma/Infiniband.cc @@ -76,13 +76,18 @@ Infiniband::Infiniband(CephContext *cct, const std::string &device_name, uint8_t max_recv_wr = device->device_attr->max_srq_wr; if (max_recv_wr > cct->_conf->ms_async_rdma_receive_buffers) { - ldout(cct, 0) << __func__ << " max allowed receive buffers is " << max_recv_wr << " use this instead." << dendl; max_recv_wr = cct->_conf->ms_async_rdma_receive_buffers; + ldout(cct, 0) << __func__ << " assigning: " << max_recv_wr << " receive buffers" << dendl; + } else { + ldout(cct, 0) << __func__ << " using the max allowed receive buffers: " << max_recv_wr << dendl; } + max_send_wr = device->device_attr->max_qp_wr; if (max_send_wr > cct->_conf->ms_async_rdma_send_buffers) { - ldout(cct, 0) << __func__ << " max allowed send buffers is " << max_send_wr << " use this instead." << dendl; max_send_wr = cct->_conf->ms_async_rdma_send_buffers; + ldout(cct, 0) << __func__ << " assigning: " << max_send_wr << " send buffers" << dendl; + } else { + ldout(cct, 0) << __func__ << " using the max allowed send buffers: " << max_send_wr << dendl; } ldout(cct, 1) << __func__ << " device allow " << device->device_attr->max_cqe @@ -91,9 +96,7 @@ Infiniband::Infiniband(CephContext *cct, const std::string &device_name, uint8_t memory_manager = new MemoryManager(device, pd, cct->_conf->ms_async_rdma_enable_hugepage); memory_manager->register_rx_tx( - cct->_conf->ms_async_rdma_buffer_size, - cct->_conf->ms_async_rdma_receive_buffers, - cct->_conf->ms_async_rdma_send_buffers); + cct->_conf->ms_async_rdma_buffer_size, max_recv_wr, max_send_wr); srq = create_shared_receive_queue(max_recv_wr, MAX_SHARED_RX_SGE_COUNT); post_channel_cluster(); @@ -157,6 +160,8 @@ int Infiniband::QueuePair::init() qp = ibv_create_qp(pd, &qpia); if (qp == NULL) { lderr(cct) << __func__ << " failed to create queue pair" << cpp_strerror(errno) << dendl; + lderr(cct) << __func__ << " try reducing ms_async_rdma_receive_buffers or" + " ms_async_rdma_send_buffers" << dendl; return -1; } -- 2.39.5