From f3b46d13df29a69add5b1711a91ac8fafabd197a Mon Sep 17 00:00:00 2001
From: Haomai Wang <haomai@xsky.com>
Date: Fri, 17 Jun 2016 14:10:59 +0800
Subject: [PATCH] msg/async: set socket priority for heartbeat socket

To prioritize the heartbeat packets in the the diffserv-aware device.

Learn/copy from https://github.com/ceph/ceph/pull/3353

Signed-off-by: Haomai Wang <haomai@xsky.com>
---
 src/msg/async/AsyncConnection.cc |  2 ++
 src/msg/async/net_handler.cc     | 28 ++++++++++++++++++++++++++++
 src/msg/async/net_handler.h      |  1 +
 3 files changed, 31 insertions(+)

diff --git a/src/msg/async/AsyncConnection.cc b/src/msg/async/AsyncConnection.cc
index 7e486d30be0..18f5d2ec46c 100644
--- a/src/msg/async/AsyncConnection.cc
+++ b/src/msg/async/AsyncConnection.cc
@@ -1008,6 +1008,7 @@ ssize_t AsyncConnection::_process_connection()
         }
 
         center->delete_file_event(sd, EVENT_WRITABLE);
+        net.set_priority(sd, async_msgr->get_socket_priority());
         ldout(async_msgr->cct, 10) << __func__ << " connect successfully, ready to send banner" << dendl;
 
         bufferlist bl;
@@ -1306,6 +1307,7 @@ ssize_t AsyncConnection::_process_connection()
           goto fail;
 
         net.set_socket_options(sd);
+        net.set_priority(sd, async_msgr->get_socket_priority());
 
         bl.append(CEPH_BANNER, strlen(CEPH_BANNER));
 
diff --git a/src/msg/async/net_handler.cc b/src/msg/async/net_handler.cc
index 9dc698ea0bc..ee196567605 100644
--- a/src/msg/async/net_handler.cc
+++ b/src/msg/async/net_handler.cc
@@ -16,6 +16,7 @@
 
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <netinet/ip.h>
 #include <netinet/tcp.h>
 #include <arpa/inet.h>
 
@@ -102,6 +103,33 @@ void NetHandler::set_socket_options(int sd)
 #endif
 }
 
+void NetHandler::set_priority(int sd, int prio)
+{
+  if (prio >= 0) {
+    int r = -1;
+#ifdef IPTOS_CLASS_CS6
+    int iptos = IPTOS_CLASS_CS6;
+    r = ::setsockopt(sd, IPPROTO_IP, IP_TOS, &iptos, sizeof(iptos));
+    if (r < 0) {
+      ldout(cct, 0) << __func__ << " couldn't set IP_TOS to " << iptos
+                    << ": " << cpp_strerror(errno) << dendl;
+    }
+#endif
+#if defined(SO_PRIORITY) 
+    // setsockopt(IPTOS_CLASS_CS6) sets the priority of the socket as 0.
+    // See http://goo.gl/QWhvsD and http://goo.gl/laTbjT
+    // We need to call setsockopt(SO_PRIORITY) after it.
+#if defined(__linux__)
+    r = ::setsockopt(sd, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio));
+#endif
+    if (r < 0) {
+      ldout(cct, 0) << __func__ << " couldn't set SO_PRIORITY to " << prio
+                    << ": " << cpp_strerror(errno) << dendl;
+    }
+#endif
+  }
+}
+
 int NetHandler::generic_connect(const entity_addr_t& addr, bool nonblock)
 {
   int ret;
diff --git a/src/msg/async/net_handler.h b/src/msg/async/net_handler.h
index 980e009f7b8..499d8fb9e6c 100644
--- a/src/msg/async/net_handler.h
+++ b/src/msg/async/net_handler.h
@@ -40,6 +40,7 @@ namespace ceph {
      */
     int reconnect(const entity_addr_t &addr, int sd);
     int nonblock_connect(const entity_addr_t &addr);
+    void set_priority(int sd, int priority);
   };
 }
 
-- 
2.47.3