msg/async/rdma: Pass specific ConnMgr info in constructor

Pass specific information for RDMAConnTCP (and later for RDMAConnCM) in
the constructor.
This makes set_accept_fd() redundant.

Issue: 995322
Change-Id: I10b4a626f180858465a378e886a238fe187180d5
Signed-off-by: Amir Vadai <amir@vadai.me>

diff --git a/src/msg/async/rdma/RDMAConnTCP.h b/src/msg/async/rdma/RDMAConnTCP.h
index 39d96d5de13b745b4edae56979866440a618a755..928bb0bdf5d8f7754f2d5ab63b7de20e10ee7ab3 100644 (file)
--- a/src/msg/async/rdma/RDMAConnTCP.h
+++ b/src/msg/async/rdma/RDMAConnTCP.h
@@ -27,6 +27,10 @@
 class RDMAWorker;
 class RDMADispatcher;
 
+struct RDMAConnTCPInfo {
+  int sd;
+};
+
 class RDMAConnTCP : public RDMAConnMgr {
   class C_handle_connection : public EventCallback {
     RDMAConnTCP *cst;
@@ -57,7 +61,8 @@ class RDMAConnTCP : public RDMAConnMgr {
 
  public:
   RDMAConnTCP(CephContext *cct, RDMAConnectedSocketImpl *sock,
-             Infiniband* ib, RDMADispatcher* s, RDMAWorker *w);
+             Infiniband* ib, RDMADispatcher* s, RDMAWorker *w,
+             void *info);
   virtual ~RDMAConnTCP();
 
   virtual ostream &print(ostream &out) const override;

diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
index 83c8b5387b9d23112ee1742c35856cb073212737..34ea63279ee0eda26700fc9c33591d8674e3b87a 100644 (file)
--- a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
+++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
@@ -32,11 +32,11 @@ RDMAConnMgr::RDMAConnMgr(CephContext *cct, RDMAConnectedSocketImpl *sock,
 }
 
 RDMAConnectedSocketImpl::RDMAConnectedSocketImpl(CephContext *cct, Infiniband* ib, RDMADispatcher* s,
-                                                RDMAWorker *w)
+                                                RDMAWorker *w, void *info)
   : cct(cct), infiniband(ib), dispatcher(s), worker(w),
     error(0), lock("RDMAConnectedSocketImpl::lock")
 {
-  cmgr = new RDMAConnTCP(cct, this, ib, s, w);
+    cmgr = new RDMAConnTCP(cct, this, ib, s, w, info);
 }
 
 QueuePair *RDMAConnectedSocketImpl::create_queue_pair(Device *d, int p)
@@ -52,7 +52,8 @@ QueuePair *RDMAConnectedSocketImpl::create_queue_pair(Device *d, int p)
 }
 
 RDMAConnTCP::RDMAConnTCP(CephContext *cct, RDMAConnectedSocketImpl *sock,
-                        Infiniband* ib, RDMADispatcher* s, RDMAWorker *w)
+                        Infiniband* ib, RDMADispatcher* s, RDMAWorker *w,
+                        void *_info)
   : RDMAConnMgr(cct, sock, ib, s, w), con_handler(new C_handle_connection(this))
 {
   Device *ibdev = ib->get_device(cct->_conf->ms_async_rdma_device_name.c_str());
@@ -71,6 +72,16 @@ RDMAConnTCP::RDMAConnTCP(CephContext *cct, RDMAConnectedSocketImpl *sock,
   my_msg.peer_qpn = 0;
   my_msg.gid = ibdev->get_gid(ibport);
   socket->register_qp(qp);
+
+  if (_info) {
+    RDMAConnTCPInfo *info = (struct RDMAConnTCPInfo *)_info;
+
+    tcp_fd = info->sd;
+    is_server = true;
+    worker->center.submit_to(worker->center.get_id(), [this]() {
+                            worker->center.create_file_event(tcp_fd, EVENT_READABLE, con_handler);
+                            }, true);
+  }
 }
 
 void RDMAConnectedSocketImpl::register_qp(QueuePair *qp)
@@ -671,12 +682,3 @@ void RDMAConnectedSocketImpl::fault()
   cmgr->connected = 1;
   notify();
 }
-
-void RDMAConnTCP::set_accept_fd(int sd)
-{
-  tcp_fd = sd;
-  is_server = true;
-  worker->center.submit_to(worker->center.get_id(), [this]() {
-                          worker->center.create_file_event(tcp_fd, EVENT_READABLE, con_handler);
-                          }, true);
-}

diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.h b/src/msg/async/rdma/RDMAConnectedSocketImpl.h
index d48334b2eeb9d5ebccf9a50d924dc18f04f022f2..857bef5c495d0f44b0d56628037f3b0961bd64fa 100644 (file)
--- a/src/msg/async/rdma/RDMAConnectedSocketImpl.h
+++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.h
@@ -52,7 +52,6 @@ class RDMAConnMgr {
 
   virtual void cleanup() = 0;
   virtual int try_connect(const entity_addr_t&, const SocketOptions &opt) = 0;
-  virtual void set_accept_fd(int sd) = 0;
 
   void post_read();
 
@@ -99,7 +98,7 @@ class RDMAConnectedSocketImpl : public ConnectedSocketImpl {
   uint32_t remote_qpn = 0;
 
   RDMAConnectedSocketImpl(CephContext *cct, Infiniband* ib, RDMADispatcher* s,
-                          RDMAWorker *w);
+                          RDMAWorker *w, void *info = nullptr);
   virtual ~RDMAConnectedSocketImpl();
 
   ostream &print(ostream &out) const {
@@ -128,7 +127,6 @@ class RDMAConnectedSocketImpl : public ConnectedSocketImpl {
   void notify();
 
   QueuePair *create_queue_pair(Device *d, int p);
-  void set_accept_fd(int sd) {cmgr->set_accept_fd(sd); };
   int try_connect(const entity_addr_t &sa, const SocketOptions &opt) { return cmgr->try_connect(sa, opt); };
 };
 inline ostream& operator<<(ostream& out, const RDMAConnectedSocketImpl &s)

diff --git a/src/msg/async/rdma/RDMAServerSocketImpl.cc b/src/msg/async/rdma/RDMAServerSocketImpl.cc
index f2b6b79b4233ef18fa6f84a0f8245b527d5502bc..1c5f170b4d0fea6fe141bd393011bbdfceb7cc8f 100644 (file)
--- a/src/msg/async/rdma/RDMAServerSocketImpl.cc
+++ b/src/msg/async/rdma/RDMAServerSocketImpl.cc
@@ -114,8 +114,8 @@ int RDMAServerConnTCP::accept(ConnectedSocket *sock, const SocketOptions &opt, e
 
   RDMAConnectedSocketImpl *server;
   //Worker* w = dispatcher->get_stack()->get_worker();
-  server = new RDMAConnectedSocketImpl(cct, infiniband, dispatcher, dynamic_cast<RDMAWorker*>(w));
-  server->set_accept_fd(sd);
+  RDMAConnTCPInfo conn_info = { sd };
+  server = new RDMAConnectedSocketImpl(cct, infiniband, dispatcher, dynamic_cast<RDMAWorker*>(w), &conn_info);
   ldout(cct, 20) << __func__ << " accepted a new QP, tcp_fd: " << sd << dendl;
   std::unique_ptr<RDMAConnectedSocketImpl> csi(server);
   *sock = ConnectedSocket(std::move(csi));