msg/async: significantly reduce minimal memory usage of AsyncConnections

AsyncConnection class instance weights over 18 KB in size, and most of it
is used by msgvec structure, which weights 16 KB. Reduce it to 4KB by reducing
number of iovecs in this array down to 256 (= 1024/4). This won't cause
noticeable perf decrease, as even under extreme loads, we exceed 256 used
iovecs in only less than 1% of all write calls. For high-density nodes and
large clusters, savings will exceed hundreds of megabytes per node.

Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>

diff --git a/src/msg/async/AsyncConnection.cc b/src/msg/async/AsyncConnection.cc
index 9398e915c2e5e56345317e4a121ebcbb6c2d702e..fffb144d3de1c747de7e2966ecdb57c6bb131962 100644 (file)
--- a/src/msg/async/AsyncConnection.cc
+++ b/src/msg/async/AsyncConnection.cc
@@ -366,7 +366,7 @@ ssize_t AsyncConnection::_try_send(bufferlist &send_bl, bool send, bool more)
   uint64_t left_pbrs = outcoming_bl.buffers().size();
   while (left_pbrs) {
     struct msghdr msg;
-    uint64_t size = MIN(left_pbrs, IOV_MAX);
+    uint64_t size = MIN(left_pbrs, ASYNC_IOV_MAX);
     left_pbrs -= size;
     memset(&msg, 0, sizeof(msg));
     msg.msg_iovlen = 0;

diff --git a/src/msg/async/AsyncConnection.h b/src/msg/async/AsyncConnection.h
index 049c2afee15a650fe590b2d912291df4964facee..c1c7051c1907ce2982ab49c193af81a686862f4f 100644 (file)
--- a/src/msg/async/AsyncConnection.h
+++ b/src/msg/async/AsyncConnection.h
@@ -36,6 +36,8 @@ using namespace std;
 
 class AsyncMessenger;
 
+static const int ASYNC_IOV_MAX = (IOV_MAX >= 1024 ? IOV_MAX / 4 : IOV_MAX);
+
 /*
  * AsyncConnection maintains a logic session between two endpoints. In other
  * word, a pair of addresses can find the only AsyncConnection. AsyncConnection
@@ -254,7 +256,7 @@ class AsyncConnection : public Connection {
   EventCallbackRef connect_handler;
   EventCallbackRef local_deliver_handler;
   EventCallbackRef wakeup_handler;
-  struct iovec msgvec[IOV_MAX];
+  struct iovec msgvec[ASYNC_IOV_MAX];
   char *recv_buf;
   uint32_t recv_max_prefetch;
   uint32_t recv_start;