msg/async: bunch of fixes

1. Use MSG_MORE in sendmsg to reduce number of actual sends and pass hint
   to do_sendmsg in callers.
2. Move restore_sigpipe() outside of loop in do_sendmsg().
3. restore_sigpipe when hit the error condition in do_sendmsg().
3. When sent_bytes >= outcoming_bl.length, don't swap outgoing_bl with
   bl and wait for bl to get out of scope, just clear out outgoing_bl.
4. Don't swap empty outcoming_bl with bl, always claim_append(bl).

Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>

diff --git a/src/msg/async/AsyncConnection.cc b/src/msg/async/AsyncConnection.cc
index 0a6e1dad1095e3e8c3d9904ee9feecde3a5e257a..9c1b250fbcba53fd1c2d787e601710c0cbe47f02 100644 (file)
--- a/src/msg/async/AsyncConnection.cc
+++ b/src/msg/async/AsyncConnection.cc
@@ -304,9 +304,9 @@ ssize_t AsyncConnection::do_sendmsg(struct msghdr &msg, unsigned len, bool more)
   while (len > 0) {
     ssize_t r;
 #if defined(MSG_NOSIGNAL)
-    r = ::sendmsg(sd, &msg, MSG_NOSIGNAL);
+    r = ::sendmsg(sd, &msg, MSG_NOSIGNAL | (more ? MSG_MORE : 0));
 #else
-    r = ::sendmsg(sd, &msg, 0);
+    r = ::sendmsg(sd, &msg, (more ? MSG_MORE : 0));
 #endif /* defined(MSG_NOSIGNAL) */
 
     if (r == 0) {
@@ -318,6 +318,7 @@ ssize_t AsyncConnection::do_sendmsg(struct msghdr &msg, unsigned len, bool more)
         break;
       } else {
         ldout(async_msgr->cct, 1) << __func__ << " sendmsg error: " << cpp_strerror(errno) << dendl;
+        restore_sigpipe();
         return r;
       }
     }
@@ -339,8 +340,8 @@ ssize_t AsyncConnection::do_sendmsg(struct msghdr &msg, unsigned len, bool more)
         break;
       }
     }
-    restore_sigpipe();
   }
+  restore_sigpipe();
   return (ssize_t)len;
 }
 
@@ -350,10 +351,7 @@ ssize_t AsyncConnection::_try_send(bufferlist &send_bl, bool send)
 {
   ldout(async_msgr->cct, 20) << __func__ << " send bl length is " << send_bl.length() << dendl;
   if (send_bl.length()) {
-    if (outcoming_bl.length())
-      outcoming_bl.claim_append(send_bl);
-    else
-      outcoming_bl.swap(send_bl);
+    outcoming_bl.claim_append(send_bl);
   }
 
   if (!send)
@@ -386,7 +384,7 @@ ssize_t AsyncConnection::_try_send(bufferlist &send_bl, bool send)
       size--;
     }
 
-    ssize_t r = do_sendmsg(msg, msglen, false);
+    ssize_t r = do_sendmsg(msg, msglen, left_pbrs);
     if (r < 0)
       return r;
 
@@ -404,9 +402,12 @@ ssize_t AsyncConnection::_try_send(bufferlist &send_bl, bool send)
   // trim already sent for outcoming_bl
   if (sent_bytes) {
     bufferlist bl;
-    if (sent_bytes < outcoming_bl.length())
+    if (sent_bytes < outcoming_bl.length()) {
       outcoming_bl.splice(sent_bytes, outcoming_bl.length()-sent_bytes, &bl);
-    bl.swap(outcoming_bl);
+      bl.swap(outcoming_bl);
+    } else {
+      outcoming_bl.clear();
+    }
   }
 
   ldout(async_msgr->cct, 20) << __func__ << " sent bytes " << sent_bytes