From d596ccf968c83e3407b56e3faa1bd0601102c5f4 Mon Sep 17 00:00:00 2001
From: sageweil <sageweil@29311d96-e01e-0410-9327-a35deaab8ce9>
Date: Fri, 31 Aug 2007 03:08:00 +0000
Subject: [PATCH] use sendmsg in simplemessenger

git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@1755 29311d96-e01e-0410-9327-a35deaab8ce9
---
 trunk/ceph/msg/SimpleMessenger.cc | 73 +++++++++++++++++++++----------
 1 file changed, 51 insertions(+), 22 deletions(-)

diff --git a/trunk/ceph/msg/SimpleMessenger.cc b/trunk/ceph/msg/SimpleMessenger.cc
index 0eae60c4f8338..dfcd505c73833 100644
--- a/trunk/ceph/msg/SimpleMessenger.cc
+++ b/trunk/ceph/msg/SimpleMessenger.cc
@@ -667,22 +667,26 @@ int Rank::Pipe::write_message(Message *m)
 	    << " in " << env->nchunks
             << dendl;
   
+  // set up msghdr and iovecs
+  struct msghdr msg;
+  memset(&msg, 0, sizeof(msg));
+  struct iovec msgvec[1 + blist.buffers().size() + env->nchunks*2];  // conservative upper bound
+  msg.msg_iov = msgvec;
+  
   // send envelope
-  int r = tcp_write( sd, (char*)env, sizeof(*env) );
-  if (r < 0) { 
-    derr(1) << "pipe(" << peer_addr << ' ' << this << ").writer error sending envelope for " << *m
-             << " to " << m->get_dest() << dendl; 
-    need_to_send_close = false;
-    return -1;
-  }
-
+  msgvec[0].iov_base = (char*)env;
+  msgvec[0].iov_len = sizeof(*env);
+  msg.msg_iovlen++;
+  
   // payload
   list<bufferptr>::const_iterator pb = blist.buffers().begin();
   list<int>::const_iterator pc = m->get_chunk_payload_at().begin();
   int b_off = 0;  // carry-over buffer offset, if any
   int bl_pos = 0; // blist pos
   int nchunks = env->nchunks;
-  while (nchunks) {
+  int32_t chunksizes[nchunks];
+
+  for (int curchunk=0; curchunk < nchunks; curchunk++) {
     // start a chunk
     int32_t size = blist.length() - bl_pos;
     if (pc != m->get_chunk_payload_at().end()) {
@@ -692,14 +696,13 @@ int Rank::Pipe::write_message(Message *m)
       pc++;
     }
     assert(size > 0);
-    dout(30) << "chunk pos " << bl_pos << " size " << size << dendl;
+    dout(30) << "chunk " << curchunk << " pos " << bl_pos << " size " << size << dendl;
 
-    r = tcp_write(sd, (char*)&size, sizeof(size));
-    if (r < 0) { 
-      derr(10) << "pipe(" << peer_addr << ' ' << this << ").writer error sending chunk len for " << *m << " to " << m->get_dest() << dendl; 
-      need_to_send_close = false;
-      return -1;
-    }
+    // chunk size
+    chunksizes[curchunk] = size;
+    msgvec[msg.msg_iovlen].iov_base = &chunksizes[curchunk];
+    msgvec[msg.msg_iovlen].iov_len = sizeof(int32_t);
+    msg.msg_iovlen++;
 
     // chunk contents
     int left = size;
@@ -711,12 +714,27 @@ int Rank::Pipe::write_message(Message *m)
 	      << " buffer len " << pb->length()
 	      << " writing " << donow 
 	      << dendl;
-      r = tcp_write(sd, pb->c_str()+b_off, donow);
-      if (r < 0) { 
-	derr(10) << "pipe(" << peer_addr << ' ' << this << ").writer error sending data chunk for " << *m << " to " << m->get_dest() << dendl; 
-	need_to_send_close = false;
-	return -1;
+
+      if (msg.msg_iovlen == IOV_MAX-1) {
+	// send what we have so far...
+	int r = sendmsg(sd, &msg, 0);
+	if (r < 0) { 
+	  assert(r == -1);
+	  derr(1) << "pipe(" << peer_addr << ' ' << this << ").writer error on sendmsg for " << *m
+		  << " to " << m->get_dest() 
+		  << ", " << strerror(errno)
+		  << dendl; 
+	  need_to_send_close = false;
+	  return -1;
+	}
+	// and restart the iov
+	msg.msg_iovlen = 0;
       }
+
+      msgvec[msg.msg_iovlen].iov_base = (void*)(pb->c_str()+b_off);
+      msgvec[msg.msg_iovlen].iov_len = donow;
+      msg.msg_iovlen++;
+
       left -= donow;
       assert(left >= 0);
       b_off += donow;
@@ -727,10 +745,21 @@ int Rank::Pipe::write_message(Message *m)
       b_off = 0;
     }
     assert(left == 0);
-    nchunks--;
   }
   assert(pb == blist.buffers().end());
   
+  // send
+  int r = sendmsg(sd, &msg, 0);
+  if (r < 0) { 
+    assert(r == -1);
+    derr(1) << "pipe(" << peer_addr << ' ' << this << ").writer error on sendmsg for " << *m
+	    << " to " << m->get_dest() 
+	    << ", " << strerror(errno)
+	    << dendl; 
+    need_to_send_close = false;
+    return -1;
+  }
+
   return 0;
 }
 
-- 
2.39.5