From ce4677612fdabab6e762cf0cd21914be438ce8e1 Mon Sep 17 00:00:00 2001 From: Amir Vadai Date: Thu, 30 Mar 2017 14:15:57 +0300 Subject: [PATCH] msg/async/rdma: Pass specific ConnMgr info in constructor Pass specific information for RDMAConnTCP (and later for RDMAConnCM) in the constructor. This makes set_accept_fd() redundant. Issue: 995322 Change-Id: I10b4a626f180858465a378e886a238fe187180d5 Signed-off-by: Amir Vadai --- src/msg/async/rdma/RDMAConnTCP.h | 7 ++++- src/msg/async/rdma/RDMAConnectedSocketImpl.cc | 26 ++++++++++--------- src/msg/async/rdma/RDMAConnectedSocketImpl.h | 4 +-- src/msg/async/rdma/RDMAServerSocketImpl.cc | 4 +-- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/msg/async/rdma/RDMAConnTCP.h b/src/msg/async/rdma/RDMAConnTCP.h index 39d96d5de13..928bb0bdf5d 100644 --- a/src/msg/async/rdma/RDMAConnTCP.h +++ b/src/msg/async/rdma/RDMAConnTCP.h @@ -27,6 +27,10 @@ class RDMAWorker; class RDMADispatcher; +struct RDMAConnTCPInfo { + int sd; +}; + class RDMAConnTCP : public RDMAConnMgr { class C_handle_connection : public EventCallback { RDMAConnTCP *cst; @@ -57,7 +61,8 @@ class RDMAConnTCP : public RDMAConnMgr { public: RDMAConnTCP(CephContext *cct, RDMAConnectedSocketImpl *sock, - Infiniband* ib, RDMADispatcher* s, RDMAWorker *w); + Infiniband* ib, RDMADispatcher* s, RDMAWorker *w, + void *info); virtual ~RDMAConnTCP(); virtual ostream &print(ostream &out) const override; diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc index 83c8b5387b9..34ea63279ee 100644 --- a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc +++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc @@ -32,11 +32,11 @@ RDMAConnMgr::RDMAConnMgr(CephContext *cct, RDMAConnectedSocketImpl *sock, } RDMAConnectedSocketImpl::RDMAConnectedSocketImpl(CephContext *cct, Infiniband* ib, RDMADispatcher* s, - RDMAWorker *w) + RDMAWorker *w, void *info) : cct(cct), infiniband(ib), dispatcher(s), worker(w), error(0), lock("RDMAConnectedSocketImpl::lock") { - cmgr = new RDMAConnTCP(cct, this, ib, s, w); + cmgr = new RDMAConnTCP(cct, this, ib, s, w, info); } QueuePair *RDMAConnectedSocketImpl::create_queue_pair(Device *d, int p) @@ -52,7 +52,8 @@ QueuePair *RDMAConnectedSocketImpl::create_queue_pair(Device *d, int p) } RDMAConnTCP::RDMAConnTCP(CephContext *cct, RDMAConnectedSocketImpl *sock, - Infiniband* ib, RDMADispatcher* s, RDMAWorker *w) + Infiniband* ib, RDMADispatcher* s, RDMAWorker *w, + void *_info) : RDMAConnMgr(cct, sock, ib, s, w), con_handler(new C_handle_connection(this)) { Device *ibdev = ib->get_device(cct->_conf->ms_async_rdma_device_name.c_str()); @@ -71,6 +72,16 @@ RDMAConnTCP::RDMAConnTCP(CephContext *cct, RDMAConnectedSocketImpl *sock, my_msg.peer_qpn = 0; my_msg.gid = ibdev->get_gid(ibport); socket->register_qp(qp); + + if (_info) { + RDMAConnTCPInfo *info = (struct RDMAConnTCPInfo *)_info; + + tcp_fd = info->sd; + is_server = true; + worker->center.submit_to(worker->center.get_id(), [this]() { + worker->center.create_file_event(tcp_fd, EVENT_READABLE, con_handler); + }, true); + } } void RDMAConnectedSocketImpl::register_qp(QueuePair *qp) @@ -671,12 +682,3 @@ void RDMAConnectedSocketImpl::fault() cmgr->connected = 1; notify(); } - -void RDMAConnTCP::set_accept_fd(int sd) -{ - tcp_fd = sd; - is_server = true; - worker->center.submit_to(worker->center.get_id(), [this]() { - worker->center.create_file_event(tcp_fd, EVENT_READABLE, con_handler); - }, true); -} diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.h b/src/msg/async/rdma/RDMAConnectedSocketImpl.h index d48334b2eeb..857bef5c495 100644 --- a/src/msg/async/rdma/RDMAConnectedSocketImpl.h +++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.h @@ -52,7 +52,6 @@ class RDMAConnMgr { virtual void cleanup() = 0; virtual int try_connect(const entity_addr_t&, const SocketOptions &opt) = 0; - virtual void set_accept_fd(int sd) = 0; void post_read(); @@ -99,7 +98,7 @@ class RDMAConnectedSocketImpl : public ConnectedSocketImpl { uint32_t remote_qpn = 0; RDMAConnectedSocketImpl(CephContext *cct, Infiniband* ib, RDMADispatcher* s, - RDMAWorker *w); + RDMAWorker *w, void *info = nullptr); virtual ~RDMAConnectedSocketImpl(); ostream &print(ostream &out) const { @@ -128,7 +127,6 @@ class RDMAConnectedSocketImpl : public ConnectedSocketImpl { void notify(); QueuePair *create_queue_pair(Device *d, int p); - void set_accept_fd(int sd) {cmgr->set_accept_fd(sd); }; int try_connect(const entity_addr_t &sa, const SocketOptions &opt) { return cmgr->try_connect(sa, opt); }; }; inline ostream& operator<<(ostream& out, const RDMAConnectedSocketImpl &s) diff --git a/src/msg/async/rdma/RDMAServerSocketImpl.cc b/src/msg/async/rdma/RDMAServerSocketImpl.cc index f2b6b79b423..1c5f170b4d0 100644 --- a/src/msg/async/rdma/RDMAServerSocketImpl.cc +++ b/src/msg/async/rdma/RDMAServerSocketImpl.cc @@ -114,8 +114,8 @@ int RDMAServerConnTCP::accept(ConnectedSocket *sock, const SocketOptions &opt, e RDMAConnectedSocketImpl *server; //Worker* w = dispatcher->get_stack()->get_worker(); - server = new RDMAConnectedSocketImpl(cct, infiniband, dispatcher, dynamic_cast(w)); - server->set_accept_fd(sd); + RDMAConnTCPInfo conn_info = { sd }; + server = new RDMAConnectedSocketImpl(cct, infiniband, dispatcher, dynamic_cast(w), &conn_info); ldout(cct, 20) << __func__ << " accepted a new QP, tcp_fd: " << sd << dendl; std::unique_ptr csi(server); *sock = ConnectedSocket(std::move(csi)); -- 2.39.5