msg/Pipe: set dscp as CS6 for heartbeat socket

To prioritize the heartbeat packets in the the diffserv-aware device.
Thanks dtaht.

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index e98f4dfb26fee553d61daca8e362993f28d4110f..ba48a93ec3adcbb4b247a476e346c05d1acdcd29 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -538,7 +538,7 @@ OPTION(osd_heartbeat_addr, OPT_ADDR, entity_addr_t())
 OPTION(osd_heartbeat_interval, OPT_INT, 6)       // (seconds) how often we ping peers
 OPTION(osd_heartbeat_grace, OPT_INT, 20)         // (seconds) how long before we decide a peer has failed
 OPTION(osd_heartbeat_min_peers, OPT_INT, 10)     // minimum number of peers
-OPTION(osd_heartbeat_use_min_delay_socket, OPT_BOOL, false)  // set SO_PRIORITY of the sockets as 6(high) if true
+OPTION(osd_heartbeat_use_min_delay_socket, OPT_BOOL, false) // prio the heartbeat tcp socket and set dscp as CS6 on it if true
 
 // max number of parallel snap trims/pg
 OPTION(osd_pg_max_concurrent_snap_trims, OPT_U64, 2)

diff --git a/src/msg/simple/Pipe.cc b/src/msg/simple/Pipe.cc
index 5dabbb0d8969ba36b53d6df16ffe0c62a523f9fd..12d2b138b0f7624d08204cbc534d02e46661a7b6 100644 (file)
--- a/src/msg/simple/Pipe.cc
+++ b/src/msg/simple/Pipe.cc
@@ -13,6 +13,7 @@
  */
 
 #include <sys/socket.h>
+#include <netinet/ip.h>
 #include <netinet/tcp.h>
 #include <sys/uio.h>
 #include <limits.h>
@@ -849,7 +850,17 @@ void Pipe::set_socket_options()
 
   int prio = msgr->get_socket_priority();
   if (prio >= 0) {
-    int r = ::setsockopt(sd, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio));
+    int iptos = IPTOS_CLASS_CS6;
+    int r = ::setsockopt(sd, IPPROTO_IP, IP_TOS, &iptos, sizeof(iptos));
+    if (r < 0) {
+      ldout(msgr->cct,0) << "couldn't set IP_TOS to " << iptos
+                         << ": " << cpp_strerror(errno) << dendl;
+    }
+
+    // setsockopt(IPTOS_CLASS_CS6) sets the priority of the socket as 0.
+    // See http://goo.gl/QWhvsD and http://goo.gl/laTbjT
+    // We need to call setsockopt(SO_PRIORITY) after it.
+    r = ::setsockopt(sd, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio));
     if (r < 0) {
       ldout(msgr->cct,0) << "couldn't set SO_PRIORITY to " << prio
                          << ": " << cpp_strerror(errno) << dendl;