From dd0d97a86651cbe3b959659574b606d8bdefbaff Mon Sep 17 00:00:00 2001 From: Chunsong Feng Date: Mon, 17 Aug 2020 20:26:29 +0800 Subject: [PATCH] msg/async/rdma: use wr_id address to check valid chunk CQE's wr_id could be: 1)BEACON_WRID 2)&RDMAConnectedSocketImpl::qp 3)Chunks address start from Cluster::chunk_base When assuming qp as Chunk through CQE's wr_id, it's possible to misjudge &(qp->ib_physical_port) into Cluster::[base, end) because there're 4 bytes random data filled in the higher 4 bytes address around ib_pysical_port due to the address alignement requirement of structure member. Fix this case by checking whether wr_id value is in the allocated Chunk space. Signed-off-by: Chunsong Feng Signed-off-by: luo rixin --- src/msg/async/rdma/Infiniband.h | 4 ++++ src/msg/async/rdma/RDMAStack.cc | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h index 03e59e7ac1772..f18442e4e6929 100644 --- a/src/msg/async/rdma/Infiniband.h +++ b/src/msg/async/rdma/Infiniband.h @@ -251,6 +251,9 @@ class Infiniband { return c >= base && c < end; } + bool is_valid_chunk(const Chunk* c) const { + return c >= chunk_base && c < chunk_base + num_chunk; + } MemoryManager& manager; uint32_t buffer_size; uint32_t num_chunk = 0; @@ -346,6 +349,7 @@ class Infiniband { void return_tx(std::vector &chunks); int get_send_buffers(std::vector &c, size_t bytes); bool is_tx_buffer(const char* c) { return send->is_my_buffer(c); } + bool is_valid_chunk(const Chunk* c) { return send->is_valid_chunk(c); } Chunk *get_tx_chunk_by_buffer(const char *c) { return send->get_chunk_by_buffer(c); } diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc index 9d2dd8027530b..c17150227bcd4 100644 --- a/src/msg/async/rdma/RDMAStack.cc +++ b/src/msg/async/rdma/RDMAStack.cc @@ -545,7 +545,7 @@ void RDMADispatcher::handle_tx_event(ibv_wc *cqe, int n) //TX completion may come either from // 1) regular send message, WCE wr_id points to chunk // 2) 'fin' message, wr_id points to the QP - if (ib->get_memory_manager()->is_tx_buffer(chunk->buffer)) { + if (ib->get_memory_manager()->is_valid_chunk(chunk)) { tx_chunks.push_back(chunk); } else if (reinterpret_cast(response->wr_id)->get_local_qp_number() == response->qp_num ) { ldout(cct, 1) << __func__ << " sending of the disconnect msg completed" << dendl; -- 2.39.5