From: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
Date: Mon, 8 Feb 2016 12:21:55 +0000 (+0100)
Subject: msg/async: significantly reduce minimal memory usage of AsyncConnections
X-Git-Tag: v10.1.0~404^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=69d98b6563e7aa69c610f26a6d2321a98d90002e;p=ceph.git

msg/async: significantly reduce minimal memory usage of AsyncConnections

AsyncConnection class instance weights over 18 KB in size, and most of it
is used by msgvec structure, which weights 16 KB. Reduce it to 4KB by reducing
number of iovecs in this array down to 256 (= 1024/4). This won't cause
noticeable perf decrease, as even under extreme loads, we exceed 256 used
iovecs in only less than 1% of all write calls. For high-density nodes and
large clusters, savings will exceed hundreds of megabytes per node.

Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
---

diff --git a/src/msg/async/AsyncConnection.cc b/src/msg/async/AsyncConnection.cc
index 9398e915c2e5..fffb144d3de1 100644
--- a/src/msg/async/AsyncConnection.cc
+++ b/src/msg/async/AsyncConnection.cc
@@ -366,7 +366,7 @@ ssize_t AsyncConnection::_try_send(bufferlist &send_bl, bool send, bool more)
   uint64_t left_pbrs = outcoming_bl.buffers().size();
   while (left_pbrs) {
     struct msghdr msg;
-    uint64_t size = MIN(left_pbrs, IOV_MAX);
+    uint64_t size = MIN(left_pbrs, ASYNC_IOV_MAX);
     left_pbrs -= size;
     memset(&msg, 0, sizeof(msg));
     msg.msg_iovlen = 0;
diff --git a/src/msg/async/AsyncConnection.h b/src/msg/async/AsyncConnection.h
index 049c2afee15a..c1c7051c1907 100644
--- a/src/msg/async/AsyncConnection.h
+++ b/src/msg/async/AsyncConnection.h
@@ -36,6 +36,8 @@ using namespace std;
 
 class AsyncMessenger;
 
+static const int ASYNC_IOV_MAX = (IOV_MAX >= 1024 ? IOV_MAX / 4 : IOV_MAX);
+
 /*
  * AsyncConnection maintains a logic session between two endpoints. In other
  * word, a pair of addresses can find the only AsyncConnection. AsyncConnection
@@ -254,7 +256,7 @@ class AsyncConnection : public Connection {
   EventCallbackRef connect_handler;
   EventCallbackRef local_deliver_handler;
   EventCallbackRef wakeup_handler;
-  struct iovec msgvec[IOV_MAX];
+  struct iovec msgvec[ASYNC_IOV_MAX];
   char *recv_buf;
   uint32_t recv_max_prefetch;
   uint32_t recv_start;