From: Sage Weil <sage@inktank.com>
Date: Thu, 28 Feb 2013 21:01:27 +0000 (-0800)
Subject: msg/Pipe: allow tuning of TCP receive buffer size
X-Git-Tag: v0.59~65
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c47c02dd87064032762d6a79f58b441f624fc8ed;p=ceph.git

msg/Pipe: allow tuning of TCP receive buffer size

Performance tests on high-end machines have indicated the Linux autotuning
of the receive buffer sizes can cause throughput collapse.  See bug
#2100, and this email discussion:

   http://marc.info/?l=ceph-devel&m=133009796706284&w=2

Initially default to 0, which leaves us with the default.  We may adjust
the default in the future.

Tested-by: Jim Schutt <jaschut@sandia.gov>
Signed-off-by: Sage Weil <sage@inktank.com>
Reviewed-by: Greg Farnum <greg@inktank.com>
---

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 76135dcdc894..278642bab466 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -101,6 +101,7 @@ OPTION(heartbeat_inject_failure, OPT_INT, 0)    // force an unhealthy heartbeat
 OPTION(perf, OPT_BOOL, true)       // enable internal perf counters
 
 OPTION(ms_tcp_nodelay, OPT_BOOL, true)
+OPTION(ms_tcp_rcvbuf, OPT_INT, 0)
 OPTION(ms_initial_backoff, OPT_DOUBLE, .2)
 OPTION(ms_max_backoff, OPT_DOUBLE, 15.0)
 OPTION(ms_nocrc, OPT_BOOL, false)
diff --git a/src/msg/Pipe.cc b/src/msg/Pipe.cc
index b9a9c3e93c92..f5965be79e2e 100644
--- a/src/msg/Pipe.cc
+++ b/src/msg/Pipe.cc
@@ -722,6 +722,14 @@ void Pipe::set_socket_options()
       ldout(msgr->cct,0) << "couldn't set TCP_NODELAY: " << cpp_strerror(r) << dendl;
     }
   }
+  if (msgr->cct->_conf->ms_tcp_rcvbuf) {
+    int size = msgr->cct->_conf->ms_tcp_rcvbuf;
+    int r = ::setsockopt(sd, SOL_SOCKET, SO_RCVBUF, (void*)&size, sizeof(size));
+    if (r < 0)  {
+      r = -errno;
+      ldout(msgr->cct,0) << "couldn't set SO_RCVBUF to " << size << ": " << cpp_strerror(r) << dendl;
+    }
+  }
 }
 
 int Pipe::connect()