xio: prepare for splitting XioMsg - methods will access members that should go to...

data members that are intended to be in the base class should be accessed
only thru a method (next, the method will be virtual in the base class)

Note: XioMsg is going to be split into two classes:
1. a base class for any xio_msg message
2. a derived class for xio_msg objects that contains Ceph messages
(than will be able to use the base class for command messages such as
ack, keepallive that xio doesn't support at the moment)

Signed-off-by: Avner BenHanoch <avnerb@mellanox.com>

diff --git a/src/msg/xio/XioConnection.cc b/src/msg/xio/XioConnection.cc
index 8543f0ced3fd6babb721b25b6beb5576850b059d..8049da4a21c0b933765bc85c2a27d75e82ff080f 100644 (file)
--- a/src/msg/xio/XioConnection.cc
+++ b/src/msg/xio/XioConnection.cc
@@ -488,7 +488,7 @@ int XioConnection::on_ow_msg_send_complete(struct xio_session *session,
 void XioConnection::msg_send_fail(XioMsg *xmsg, int code)
 {
   ldout(msgr->cct,2) << "xio_send_msg FAILED xcon: " << this <<
-    " xmsg: " << &xmsg->req_0.msg << " code=" << code <<
+    " xmsg: " << xmsg->get_xio_msg() << " code=" << code <<
     " (" << xio_strerror(code) << ")" << dendl;
   /* return refs taken for each xio_msg */
   xmsg->put_msg_refs();
@@ -562,7 +562,7 @@ int XioConnection::discard_input_queue(uint32_t flags)
        xmsg = static_cast<XioMsg*>(xs);
        deferred_q.erase(q_iter);
        // release once for each chained xio_msg
-       xmsg->put(xmsg->hdr.msg_cnt);
+       xmsg->put(xmsg->get_msg_count());
        break;
       case XioSubmit::INCOMING_MSG_RELEASE:
        deferred_q.erase(q_iter);

diff --git a/src/msg/xio/XioMessenger.cc b/src/msg/xio/XioMessenger.cc
index 8b42832108ced9947d609e82e32cff902c3d46d6..e354458ae2ccff3fc6a58b9a29623120ab1878f4 100644 (file)
--- a/src/msg/xio/XioMessenger.cc
+++ b/src/msg/xio/XioMessenger.cc
@@ -861,7 +861,7 @@ int XioMessenger::_send_message_impl(Message* m, XioConnection* xcon)
 
   ldout(cct,4) << __func__ << " " << m << " new XioMsg " << xmsg
        << " tag " << (int)xmsg->hdr.tag
-       << " req_0 " << &xmsg->req_0.msg << " msg type " << m->get_type()
+       << " req_0 " << xmsg->get_xio_msg() << " msg type " << m->get_type()
        << " features: " << xcon->get_features()
        << " conn " << xcon->conn << " sess " << xcon->session << dendl;
 
@@ -878,7 +878,7 @@ int XioMessenger::_send_message_impl(Message* m, XioConnection* xcon)
     }
   }
 
-  struct xio_msg *req = &xmsg->req_0.msg;
+  struct xio_msg *req = xmsg->get_xio_msg();
   struct xio_iovec_ex *msg_iov = req->out.pdata_iov.sglist;
 
   if (magic & (MSG_MAGIC_XIO)) {
@@ -907,14 +907,14 @@ int XioMessenger::_send_message_impl(Message* m, XioConnection* xcon)
   xio_place_buffers(data, xmsg, req, msg_iov, req_size, ex_cnt, msg_off,
                    req_off, BUFFER_DATA);
   ldout(cct,10) << "ex_cnt " << ex_cnt << ", req_off " << req_off
-    << ", msg_cnt " << xmsg->hdr.msg_cnt << dendl;
+    << ", msg_cnt " << xmsg->get_msg_count() << dendl;
 
   /* finalize request */
   if (msg_off)
     req->out.pdata_iov.nents = msg_off;
 
   /* fixup first msg */
-  req = &xmsg->req_0.msg;
+  req = xmsg->get_xio_msg();
 
   const std::list<buffer::ptr>& header = xmsg->hdr.get_bl().buffers();
   assert(header.size() == 1); /* XXX */
@@ -923,10 +923,10 @@ int XioMessenger::_send_message_impl(Message* m, XioConnection* xcon)
   req->out.header.iov_len = pb->length();
 
   /* deliver via xio, preserve ordering */
-  if (xmsg->hdr.msg_cnt > 1) {
-    struct xio_msg *head = &xmsg->req_0.msg;
+  if (xmsg->get_msg_count() > 1) {
+    struct xio_msg *head = xmsg->get_xio_msg();
     struct xio_msg *tail = head;
-    for (req_off = 0; ((unsigned) req_off) < xmsg->hdr.msg_cnt-1; ++req_off) {
+    for (req_off = 0; ((unsigned) req_off) < xmsg->get_msg_count()-1; ++req_off) {
       req = &xmsg->req_arr[req_off].msg;
 assert(!req->in.pdata_iov.nents);
 assert(req->out.pdata_iov.nents || !nbuffers);

diff --git a/src/msg/xio/XioMsg.cc b/src/msg/xio/XioMsg.cc
index 76c6392a89ece5d38a5d10ea05713b09a5a147ea..22b340029e2f40321ee703f3a818ecf7bf5dd4bc 100644 (file)
--- a/src/msg/xio/XioMsg.cc
+++ b/src/msg/xio/XioMsg.cc
@@ -43,3 +43,8 @@ int XioDispatchHook::release_msgs()
   assert(hdr_buffers.size() == 1); /* accelio header is small without scatter gather */
   return hdr_buffers.begin()->length();
 }
+
+void XioMsg::print_debug(CephContext *cct, const char *tag) const {
+  print_xio_msg_hdr(cct, tag, hdr, get_xio_msg());
+  print_ceph_msg(cct, tag, m);
+}

diff --git a/src/msg/xio/XioMsg.h b/src/msg/xio/XioMsg.h
index c0141a85e2364313da7d0ec12884e6e6cbcb7a5a..1e1fa889692b6547220547a5bba9c489b9622a5e 100644 (file)
--- a/src/msg/xio/XioMsg.h
+++ b/src/msg/xio/XioMsg.h
@@ -215,6 +215,11 @@ public:
       xcon->get();
     }
 
+  void print_debug(CephContext *cct, const char *tag) const;
+  const struct xio_msg * get_xio_msg() const {return &req_0.msg;}
+  struct xio_msg * get_xio_msg() {return &req_0.msg;}
+  size_t get_msg_count() const {return hdr.msg_cnt;}
+
   XioMsg* get() { nrefs.inc(); return this; };
 
   void put(int n) {
@@ -231,7 +236,7 @@ public:
   }
 
   void put_msg_refs() {
-    put(hdr.msg_cnt);
+    put(get_msg_count());
   }
 
   void alloc_trailers(int cnt) {
@@ -247,7 +252,7 @@ public:
   ~XioMsg()
     {
       if (unlikely(!!req_arr)) {
-       for (unsigned int ix = 0; ix < hdr.msg_cnt-1; ++ix) {
+       for (unsigned int ix = 0; ix < get_msg_count()-1; ++ix) {
          xio_msg_ex* xreq = &(req_arr[ix]);
          xreq->~xio_msg_ex();
        }

diff --git a/src/msg/xio/XioPortal.h b/src/msg/xio/XioPortal.h
index f5f5b48affda5f317e25260e92ce2bf83f394608..a1a141ce03c4498a62219aef9b509d6d098a2799 100644 (file)
--- a/src/msg/xio/XioPortal.h
+++ b/src/msg/xio/XioPortal.h
@@ -269,18 +269,17 @@ public:
                 * on Accelio's check on below, but this assures that
                 * all chained xio_msg are accounted) */
                xio_qdepth_high = xcon->xio_qdepth_high_mark();
-               if (unlikely((xcon->send_ctr + xmsg->hdr.msg_cnt) >
+               if (unlikely((xcon->send_ctr + xmsg->get_msg_count()) >
                             xio_qdepth_high)) {
                  requeue_all_xcon(xcon, q_iter, send_q);
                  goto restart;
                }
 
-               msg = &xmsg->req_0.msg;
+               msg = xmsg->get_xio_msg();
                code = xio_send_msg(xcon->conn, msg);
                /* header trace moved here to capture xio serial# */
                if (ldlog_p1(msgr->cct, ceph_subsys_xio, 11)) {
-                 print_xio_msg_hdr(msgr->cct, "xio_send_msg", xmsg->hdr, msg);
-                 print_ceph_msg(msgr->cct, "xio_send_msg", xmsg->m);
+                 xmsg->print_debug(msgr->cct, "xio_send_msg");
                }
                /* get the right Accelio's errno code */
                if (unlikely(code)) {
@@ -313,7 +312,7 @@ public:
                };
              } else {
                xcon->send.set(msg->timestamp); // need atomic?
-               xcon->send_ctr += xmsg->hdr.msg_cnt; // only inc if cb promised
+               xcon->send_ctr += xmsg->get_msg_count(); // only inc if cb promised
              }
              break;
            default: