msg/async: set socket priority for heartbeat socket

To prioritize the heartbeat packets in the the diffserv-aware device.

Learn/copy from https://github.com/ceph/ceph/pull/3353

Signed-off-by: Haomai Wang <haomai@xsky.com>

diff --git a/src/msg/async/AsyncConnection.cc b/src/msg/async/AsyncConnection.cc
index 7e486d30be0af0d3b8c1d58ba021ca224767f7ed..18f5d2ec46c392684d7882c775df4325e9e07e13 100644 (file)
--- a/src/msg/async/AsyncConnection.cc
+++ b/src/msg/async/AsyncConnection.cc
@@ -1008,6 +1008,7 @@ ssize_t AsyncConnection::_process_connection()
         }
 
         center->delete_file_event(sd, EVENT_WRITABLE);
+        net.set_priority(sd, async_msgr->get_socket_priority());
         ldout(async_msgr->cct, 10) << __func__ << " connect successfully, ready to send banner" << dendl;
 
         bufferlist bl;
@@ -1306,6 +1307,7 @@ ssize_t AsyncConnection::_process_connection()
           goto fail;
 
         net.set_socket_options(sd);
+        net.set_priority(sd, async_msgr->get_socket_priority());
 
         bl.append(CEPH_BANNER, strlen(CEPH_BANNER));
 

diff --git a/src/msg/async/net_handler.cc b/src/msg/async/net_handler.cc
index 9dc698ea0bcc0bc601fd63d5d58e6be32a603d33..ee1965676050e29280f8f9fb1f9c60ae207392d1 100644 (file)
--- a/src/msg/async/net_handler.cc
+++ b/src/msg/async/net_handler.cc
@@ -16,6 +16,7 @@
 
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <netinet/ip.h>
 #include <netinet/tcp.h>
 #include <arpa/inet.h>
 
@@ -102,6 +103,33 @@ void NetHandler::set_socket_options(int sd)
 #endif
 }
 
+void NetHandler::set_priority(int sd, int prio)
+{
+  if (prio >= 0) {
+    int r = -1;
+#ifdef IPTOS_CLASS_CS6
+    int iptos = IPTOS_CLASS_CS6;
+    r = ::setsockopt(sd, IPPROTO_IP, IP_TOS, &iptos, sizeof(iptos));
+    if (r < 0) {
+      ldout(cct, 0) << __func__ << " couldn't set IP_TOS to " << iptos
+                    << ": " << cpp_strerror(errno) << dendl;
+    }
+#endif
+#if defined(SO_PRIORITY) 
+    // setsockopt(IPTOS_CLASS_CS6) sets the priority of the socket as 0.
+    // See http://goo.gl/QWhvsD and http://goo.gl/laTbjT
+    // We need to call setsockopt(SO_PRIORITY) after it.
+#if defined(__linux__)
+    r = ::setsockopt(sd, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio));
+#endif
+    if (r < 0) {
+      ldout(cct, 0) << __func__ << " couldn't set SO_PRIORITY to " << prio
+                    << ": " << cpp_strerror(errno) << dendl;
+    }
+#endif
+  }
+}
+
 int NetHandler::generic_connect(const entity_addr_t& addr, bool nonblock)
 {
   int ret;

diff --git a/src/msg/async/net_handler.h b/src/msg/async/net_handler.h
index 980e009f7b8624e4064dcd32d5fef49e275e0314..499d8fb9e6cbc0358f5185d60556a8650b7531a2 100644 (file)
--- a/src/msg/async/net_handler.h
+++ b/src/msg/async/net_handler.h
@@ -40,6 +40,7 @@ namespace ceph {
      */
     int reconnect(const entity_addr_t &addr, int sd);
     int nonblock_connect(const entity_addr_t &addr);
+    void set_priority(int sd, int priority);
   };
 }