msg/Pipe: allow tuning of TCP receive buffer size

Performance tests on high-end machines have indicated the Linux autotuning
of the receive buffer sizes can cause throughput collapse.  See bug
#2100, and this email discussion:

   http://marc.info/?l=ceph-devel&m=133009796706284&w=2

Initially default to 0, which leaves us with the default.  We may adjust
the default in the future.

Tested-by: Jim Schutt <jaschut@sandia.gov>
Signed-off-by: Sage Weil <sage@inktank.com>
Reviewed-by: Greg Farnum <greg@inktank.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 76135dcdc8945350fcae3ab2efc43f4a1a29a9a2..278642bab466b3a39340692c6b7d3123dc6b5d2f 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -101,6 +101,7 @@ OPTION(heartbeat_inject_failure, OPT_INT, 0)    // force an unhealthy heartbeat
 OPTION(perf, OPT_BOOL, true)       // enable internal perf counters
 
 OPTION(ms_tcp_nodelay, OPT_BOOL, true)
+OPTION(ms_tcp_rcvbuf, OPT_INT, 0)
 OPTION(ms_initial_backoff, OPT_DOUBLE, .2)
 OPTION(ms_max_backoff, OPT_DOUBLE, 15.0)
 OPTION(ms_nocrc, OPT_BOOL, false)

diff --git a/src/msg/Pipe.cc b/src/msg/Pipe.cc
index b9a9c3e93c92dcb0a84fc66d6c23050b3f4cc436..f5965be79e2ef509fdb274daea0deec882f99894 100644 (file)
--- a/src/msg/Pipe.cc
+++ b/src/msg/Pipe.cc
@@ -722,6 +722,14 @@ void Pipe::set_socket_options()
       ldout(msgr->cct,0) << "couldn't set TCP_NODELAY: " << cpp_strerror(r) << dendl;
     }
   }
+  if (msgr->cct->_conf->ms_tcp_rcvbuf) {
+    int size = msgr->cct->_conf->ms_tcp_rcvbuf;
+    int r = ::setsockopt(sd, SOL_SOCKET, SO_RCVBUF, (void*)&size, sizeof(size));
+    if (r < 0)  {
+      r = -errno;
+      ldout(msgr->cct,0) << "couldn't set SO_RCVBUF to " << size << ": " << cpp_strerror(r) << dendl;
+    }
+  }
 }
 
 int Pipe::connect()