From 02b83fb9cc8bb58df506bc6753c2c3087743097c Mon Sep 17 00:00:00 2001 From: Avner BenHanoch Date: Thu, 14 Apr 2016 12:44:13 +0300 Subject: [PATCH] xio: prepare for splitting XioMsg - methods will access members that should go to base class data members that are intended to be in the base class should be accessed only thru a method (next, the method will be virtual in the base class) Note: XioMsg is going to be split into two classes: 1. a base class for any xio_msg message 2. a derived class for xio_msg objects that contains Ceph messages (than will be able to use the base class for command messages such as ack, keepallive that xio doesn't support at the moment) Signed-off-by: Avner BenHanoch --- src/msg/xio/XioConnection.cc | 4 ++-- src/msg/xio/XioMessenger.cc | 14 +++++++------- src/msg/xio/XioMsg.cc | 5 +++++ src/msg/xio/XioMsg.h | 9 +++++++-- src/msg/xio/XioPortal.h | 9 ++++----- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/msg/xio/XioConnection.cc b/src/msg/xio/XioConnection.cc index 8543f0ced3fd..8049da4a21c0 100644 --- a/src/msg/xio/XioConnection.cc +++ b/src/msg/xio/XioConnection.cc @@ -488,7 +488,7 @@ int XioConnection::on_ow_msg_send_complete(struct xio_session *session, void XioConnection::msg_send_fail(XioMsg *xmsg, int code) { ldout(msgr->cct,2) << "xio_send_msg FAILED xcon: " << this << - " xmsg: " << &xmsg->req_0.msg << " code=" << code << + " xmsg: " << xmsg->get_xio_msg() << " code=" << code << " (" << xio_strerror(code) << ")" << dendl; /* return refs taken for each xio_msg */ xmsg->put_msg_refs(); @@ -562,7 +562,7 @@ int XioConnection::discard_input_queue(uint32_t flags) xmsg = static_cast(xs); deferred_q.erase(q_iter); // release once for each chained xio_msg - xmsg->put(xmsg->hdr.msg_cnt); + xmsg->put(xmsg->get_msg_count()); break; case XioSubmit::INCOMING_MSG_RELEASE: deferred_q.erase(q_iter); diff --git a/src/msg/xio/XioMessenger.cc b/src/msg/xio/XioMessenger.cc index 8b42832108ce..e354458ae2cc 100644 --- a/src/msg/xio/XioMessenger.cc +++ b/src/msg/xio/XioMessenger.cc @@ -861,7 +861,7 @@ int XioMessenger::_send_message_impl(Message* m, XioConnection* xcon) ldout(cct,4) << __func__ << " " << m << " new XioMsg " << xmsg << " tag " << (int)xmsg->hdr.tag - << " req_0 " << &xmsg->req_0.msg << " msg type " << m->get_type() + << " req_0 " << xmsg->get_xio_msg() << " msg type " << m->get_type() << " features: " << xcon->get_features() << " conn " << xcon->conn << " sess " << xcon->session << dendl; @@ -878,7 +878,7 @@ int XioMessenger::_send_message_impl(Message* m, XioConnection* xcon) } } - struct xio_msg *req = &xmsg->req_0.msg; + struct xio_msg *req = xmsg->get_xio_msg(); struct xio_iovec_ex *msg_iov = req->out.pdata_iov.sglist; if (magic & (MSG_MAGIC_XIO)) { @@ -907,14 +907,14 @@ int XioMessenger::_send_message_impl(Message* m, XioConnection* xcon) xio_place_buffers(data, xmsg, req, msg_iov, req_size, ex_cnt, msg_off, req_off, BUFFER_DATA); ldout(cct,10) << "ex_cnt " << ex_cnt << ", req_off " << req_off - << ", msg_cnt " << xmsg->hdr.msg_cnt << dendl; + << ", msg_cnt " << xmsg->get_msg_count() << dendl; /* finalize request */ if (msg_off) req->out.pdata_iov.nents = msg_off; /* fixup first msg */ - req = &xmsg->req_0.msg; + req = xmsg->get_xio_msg(); const std::list& header = xmsg->hdr.get_bl().buffers(); assert(header.size() == 1); /* XXX */ @@ -923,10 +923,10 @@ int XioMessenger::_send_message_impl(Message* m, XioConnection* xcon) req->out.header.iov_len = pb->length(); /* deliver via xio, preserve ordering */ - if (xmsg->hdr.msg_cnt > 1) { - struct xio_msg *head = &xmsg->req_0.msg; + if (xmsg->get_msg_count() > 1) { + struct xio_msg *head = xmsg->get_xio_msg(); struct xio_msg *tail = head; - for (req_off = 0; ((unsigned) req_off) < xmsg->hdr.msg_cnt-1; ++req_off) { + for (req_off = 0; ((unsigned) req_off) < xmsg->get_msg_count()-1; ++req_off) { req = &xmsg->req_arr[req_off].msg; assert(!req->in.pdata_iov.nents); assert(req->out.pdata_iov.nents || !nbuffers); diff --git a/src/msg/xio/XioMsg.cc b/src/msg/xio/XioMsg.cc index 76c6392a89ec..22b340029e2f 100644 --- a/src/msg/xio/XioMsg.cc +++ b/src/msg/xio/XioMsg.cc @@ -43,3 +43,8 @@ int XioDispatchHook::release_msgs() assert(hdr_buffers.size() == 1); /* accelio header is small without scatter gather */ return hdr_buffers.begin()->length(); } + +void XioMsg::print_debug(CephContext *cct, const char *tag) const { + print_xio_msg_hdr(cct, tag, hdr, get_xio_msg()); + print_ceph_msg(cct, tag, m); +} diff --git a/src/msg/xio/XioMsg.h b/src/msg/xio/XioMsg.h index c0141a85e236..1e1fa889692b 100644 --- a/src/msg/xio/XioMsg.h +++ b/src/msg/xio/XioMsg.h @@ -215,6 +215,11 @@ public: xcon->get(); } + void print_debug(CephContext *cct, const char *tag) const; + const struct xio_msg * get_xio_msg() const {return &req_0.msg;} + struct xio_msg * get_xio_msg() {return &req_0.msg;} + size_t get_msg_count() const {return hdr.msg_cnt;} + XioMsg* get() { nrefs.inc(); return this; }; void put(int n) { @@ -231,7 +236,7 @@ public: } void put_msg_refs() { - put(hdr.msg_cnt); + put(get_msg_count()); } void alloc_trailers(int cnt) { @@ -247,7 +252,7 @@ public: ~XioMsg() { if (unlikely(!!req_arr)) { - for (unsigned int ix = 0; ix < hdr.msg_cnt-1; ++ix) { + for (unsigned int ix = 0; ix < get_msg_count()-1; ++ix) { xio_msg_ex* xreq = &(req_arr[ix]); xreq->~xio_msg_ex(); } diff --git a/src/msg/xio/XioPortal.h b/src/msg/xio/XioPortal.h index f5f5b48affda..a1a141ce03c4 100644 --- a/src/msg/xio/XioPortal.h +++ b/src/msg/xio/XioPortal.h @@ -269,18 +269,17 @@ public: * on Accelio's check on below, but this assures that * all chained xio_msg are accounted) */ xio_qdepth_high = xcon->xio_qdepth_high_mark(); - if (unlikely((xcon->send_ctr + xmsg->hdr.msg_cnt) > + if (unlikely((xcon->send_ctr + xmsg->get_msg_count()) > xio_qdepth_high)) { requeue_all_xcon(xcon, q_iter, send_q); goto restart; } - msg = &xmsg->req_0.msg; + msg = xmsg->get_xio_msg(); code = xio_send_msg(xcon->conn, msg); /* header trace moved here to capture xio serial# */ if (ldlog_p1(msgr->cct, ceph_subsys_xio, 11)) { - print_xio_msg_hdr(msgr->cct, "xio_send_msg", xmsg->hdr, msg); - print_ceph_msg(msgr->cct, "xio_send_msg", xmsg->m); + xmsg->print_debug(msgr->cct, "xio_send_msg"); } /* get the right Accelio's errno code */ if (unlikely(code)) { @@ -313,7 +312,7 @@ public: }; } else { xcon->send.set(msg->timestamp); // need atomic? - xcon->send_ctr += xmsg->hdr.msg_cnt; // only inc if cb promised + xcon->send_ctr += xmsg->get_msg_count(); // only inc if cb promised } break; default: -- 2.47.3