Revert "msg/async/rdma: Pass specific ConnMgr info in constructor"

This reverts commit ce4677612fdabab6e762cf0cd21914be438ce8e1.

Change-Id: I5098c109e7be17402ca512c25ba0c250ad610cea
Issue: 995322
Signed-off-by: Amir Vadai <amir@vadai.me>

diff --git a/src/msg/async/rdma/RDMAConnTCP.h b/src/msg/async/rdma/RDMAConnTCP.h
index 928bb0bdf5d8f7754f2d5ab63b7de20e10ee7ab3..39d96d5de13b745b4edae56979866440a618a755 100644 (file)
--- a/src/msg/async/rdma/RDMAConnTCP.h
+++ b/src/msg/async/rdma/RDMAConnTCP.h
@@ -27,10 +27,6 @@
 class RDMAWorker;
 class RDMADispatcher;
 
-struct RDMAConnTCPInfo {
-  int sd;
-};
-
 class RDMAConnTCP : public RDMAConnMgr {
   class C_handle_connection : public EventCallback {
     RDMAConnTCP *cst;
@@ -61,8 +57,7 @@ class RDMAConnTCP : public RDMAConnMgr {
 
  public:
   RDMAConnTCP(CephContext *cct, RDMAConnectedSocketImpl *sock,
-             Infiniband* ib, RDMADispatcher* s, RDMAWorker *w,
-             void *info);
+             Infiniband* ib, RDMADispatcher* s, RDMAWorker *w);
   virtual ~RDMAConnTCP();
 
   virtual ostream &print(ostream &out) const override;

diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
index 3014bbbd4be60e2a909c9b676cdd0f272c1d46b8..301cfe1c9822dba2557186de18361c9f9080c69d 100644 (file)
--- a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
+++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
@@ -32,11 +32,11 @@ RDMAConnMgr::RDMAConnMgr(CephContext *cct, RDMAConnectedSocketImpl *sock,
 }
 
 RDMAConnectedSocketImpl::RDMAConnectedSocketImpl(CephContext *cct, Infiniband* ib, RDMADispatcher* s,
-                                                RDMAWorker *w, void *info)
+                                                RDMAWorker *w)
   : cct(cct), infiniband(ib), dispatcher(s), worker(w),
     error(0), lock("RDMAConnectedSocketImpl::lock")
 {
-    cmgr = new RDMAConnTCP(cct, this, ib, s, w, info);
+  cmgr = new RDMAConnTCP(cct, this, ib, s, w);
 }
 
 QueuePair *RDMAConnectedSocketImpl::create_queue_pair(Device *d, int p)
@@ -52,8 +52,7 @@ QueuePair *RDMAConnectedSocketImpl::create_queue_pair(Device *d, int p)
 }
 
 RDMAConnTCP::RDMAConnTCP(CephContext *cct, RDMAConnectedSocketImpl *sock,
-                        Infiniband* ib, RDMADispatcher* s, RDMAWorker *w,
-                        void *_info)
+                        Infiniband* ib, RDMADispatcher* s, RDMAWorker *w)
   : RDMAConnMgr(cct, sock, ib, s, w), con_handler(new C_handle_connection(this))
 {
   Device *ibdev = ib->get_device(cct->_conf->ms_async_rdma_device_name.c_str());
@@ -72,16 +71,6 @@ RDMAConnTCP::RDMAConnTCP(CephContext *cct, RDMAConnectedSocketImpl *sock,
   my_msg.peer_qpn = 0;
   my_msg.gid = ibdev->get_gid(ibport);
   socket->register_qp(qp);
-
-  if (_info) {
-    RDMAConnTCPInfo *info = (struct RDMAConnTCPInfo *)_info;
-
-    tcp_fd = info->sd;
-    is_server = true;
-    worker->center.submit_to(worker->center.get_id(), [this]() {
-                            worker->center.create_file_event(tcp_fd, EVENT_READABLE, con_handler);
-                            }, true);
-  }
 }
 
 void RDMAConnectedSocketImpl::register_qp(QueuePair *qp)
@@ -687,3 +676,12 @@ void RDMAConnectedSocketImpl::fault()
   cmgr->connected = 1;
   notify();
 }
+
+void RDMAConnTCP::set_accept_fd(int sd)
+{
+  tcp_fd = sd;
+  is_server = true;
+  worker->center.submit_to(worker->center.get_id(), [this]() {
+                          worker->center.create_file_event(tcp_fd, EVENT_READABLE, con_handler);
+                          }, true);
+}

diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.h b/src/msg/async/rdma/RDMAConnectedSocketImpl.h
index 857bef5c495d0f44b0d56628037f3b0961bd64fa..d48334b2eeb9d5ebccf9a50d924dc18f04f022f2 100644 (file)
--- a/src/msg/async/rdma/RDMAConnectedSocketImpl.h
+++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.h
@@ -52,6 +52,7 @@ class RDMAConnMgr {
 
   virtual void cleanup() = 0;
   virtual int try_connect(const entity_addr_t&, const SocketOptions &opt) = 0;
+  virtual void set_accept_fd(int sd) = 0;
 
   void post_read();
 
@@ -98,7 +99,7 @@ class RDMAConnectedSocketImpl : public ConnectedSocketImpl {
   uint32_t remote_qpn = 0;
 
   RDMAConnectedSocketImpl(CephContext *cct, Infiniband* ib, RDMADispatcher* s,
-                          RDMAWorker *w, void *info = nullptr);
+                          RDMAWorker *w);
   virtual ~RDMAConnectedSocketImpl();
 
   ostream &print(ostream &out) const {
@@ -127,6 +128,7 @@ class RDMAConnectedSocketImpl : public ConnectedSocketImpl {
   void notify();
 
   QueuePair *create_queue_pair(Device *d, int p);
+  void set_accept_fd(int sd) {cmgr->set_accept_fd(sd); };
   int try_connect(const entity_addr_t &sa, const SocketOptions &opt) { return cmgr->try_connect(sa, opt); };
 };
 inline ostream& operator<<(ostream& out, const RDMAConnectedSocketImpl &s)

diff --git a/src/msg/async/rdma/RDMAServerSocketImpl.cc b/src/msg/async/rdma/RDMAServerSocketImpl.cc
index a9665d96edf016260e24a9a462a2cf210b466eef..d7065baeb12bc1a726c32e5362789e9170c30704 100644 (file)
--- a/src/msg/async/rdma/RDMAServerSocketImpl.cc
+++ b/src/msg/async/rdma/RDMAServerSocketImpl.cc
@@ -118,8 +118,8 @@ int RDMAServerConnTCP::accept(ConnectedSocket *sock, const SocketOptions &opt, e
 
   RDMAConnectedSocketImpl *server;
   //Worker* w = dispatcher->get_stack()->get_worker();
-  RDMAConnTCPInfo conn_info = { sd };
-  server = new RDMAConnectedSocketImpl(cct, infiniband, dispatcher, dynamic_cast<RDMAWorker*>(w), &conn_info);
+  server = new RDMAConnectedSocketImpl(cct, infiniband, dispatcher, dynamic_cast<RDMAWorker*>(w));
+  server->set_accept_fd(sd);
   ldout(cct, 20) << __func__ << " accepted a new QP, tcp_fd: " << sd << dendl;
   std::unique_ptr<RDMAConnectedSocketImpl> csi(server);
   *sock = ConnectedSocket(std::move(csi));