From 16c6dd5129f54a6f5eec7fe16719a179c176548e Mon Sep 17 00:00:00 2001
From: Greg Farnum <gfarnum@redhat.com>
Date: Tue, 3 Oct 2017 15:54:06 -0700
Subject: [PATCH] msgr: add a mechanism for Solaris to avoid dying on SIGPIPE

This is fairly clean: we define an RAII object in the Messenger.h on
Solaris, and "declare" it with a macro in the implementations. There's
no code duplication and on Linux it's just entirely compiled out.

Signed-off-by: Greg Farnum <gfarnum@redhat.com>
(cherry picked from commit cba20a95e816aaf9f0971b29b14b0be8c524b59d)

Conflicts:
    src/msg/async/PosixStack.cc - luminous #include "common/simple_spin.h" is
        missing in master (trivial resolution)
---
 src/include/sock_compat.h   |  3 ++-
 src/msg/Messenger.h         | 45 ++++++++++++++++++++++++++++++++++++-
 src/msg/async/PosixStack.cc |  4 +++-
 src/msg/simple/Pipe.cc      |  2 ++
 4 files changed, 51 insertions(+), 3 deletions(-)

diff --git a/src/include/sock_compat.h b/src/include/sock_compat.h
index 5faacc343edc6..f9dc24b1ddc6b 100644
--- a/src/include/sock_compat.h
+++ b/src/include/sock_compat.h
@@ -19,7 +19,8 @@
 # ifdef SO_NOSIGPIPE
 #  define CEPH_USE_SO_NOSIGPIPE
 # else
-#  error "Cannot block SIGPIPE!"
+#  define CEPH_USE_SIGPIPE_BLOCKER
+#  warning "Using SIGPIPE blocking instead of suppression; this is not well-tested upstream!"
 # endif
 #endif
 
diff --git a/src/msg/Messenger.h b/src/msg/Messenger.h
index 7c1a0d1fad5fe..c6dbcc17694d4 100644
--- a/src/msg/Messenger.h
+++ b/src/msg/Messenger.h
@@ -31,6 +31,7 @@ using namespace std;
 
 #include <errno.h>
 #include <sstream>
+#include <signal.h>
 
 #define SOCKET_PRIORITY_MIN_DELAY 6
 
@@ -558,11 +559,53 @@ protected:
   /**
    * @} // Subclass Interfacing
    */
+public:
+#ifdef CEPH_USE_SIGPIPE_BLOCKER
+  /**
+   * We need to disable SIGPIPE on all platforms, and if they
+   * don't give us a better mechanism (read: are on Solaris) that
+   * means blocking the signal whenever we do a send or sendmsg...
+   * That means any implementations must invoke MSGR_SIGPIPE_STOPPER in-scope
+   * whenever doing so. On most systems that's blank, but on systems where
+   * it's needed we construct an RAII object to plug and un-plug the SIGPIPE.
+   * See http://www.microhowto.info/howto/ignore_sigpipe_without_affecting_other_threads_in_a_process.html
+   */
+  struct sigpipe_stopper {
+    bool blocked;
+    sigset_t existing_mask;
+    sigset_t pipe_mask;
+    sigpipe_stopper() {
+      sigemptyset(&pipe_mask);
+      sigaddset(&pipe_mask, SIGPIPE);
+      sigset_t signals;
+      sigemptyset(&signals);
+      sigpending(&signals);
+      if (sigismember(&signals, SIGPIPE)) {
+	blocked = false;
+      } else {
+	blocked = true;
+	int r = pthread_sigmask(SIG_BLOCK, &pipe_mask, &existing_mask);
+	assert(r == 0);
+      }
+    }
+    ~sigpipe_stopper() {
+      if (blocked) {
+	struct timespec nowait{0};
+	int r = sigtimedwait(&pipe_mask, 0, &nowait);
+	assert(r == EAGAIN || r == 0);
+	r = pthread_sigmask(SIG_SETMASK, &existing_mask, 0);
+	assert(r == 0);
+      }
+    }
+  };
+#  define MSGR_SIGPIPE_STOPPER Messenger::sigpipe_stopper stopper();
+#else
+#  define MSGR_SIGPIPE_STOPPER
+#endif
   /**
    * @defgroup Dispatcher Interfacing
    * @{
    */
-public:
   /**
    * Determine whether a message can be fast-dispatched. We will
    * query each Dispatcher in sequence to determine if they are
diff --git a/src/msg/async/PosixStack.cc b/src/msg/async/PosixStack.cc
index 1bcffb469fc07..5fb975ae906ea 100644
--- a/src/msg/async/PosixStack.cc
+++ b/src/msg/async/PosixStack.cc
@@ -26,11 +26,12 @@
 
 #include "include/buffer.h"
 #include "include/str_list.h"
-#include "include/sock_compat.h"
 #include "common/errno.h"
 #include "common/strtol.h"
 #include "common/dout.h"
 #include "common/simple_spin.h"
+#include "msg/Messenger.h"
+#include "include/sock_compat.h"
 
 #define dout_subsys ceph_subsys_ms
 #undef dout_prefix
@@ -78,6 +79,7 @@ class PosixConnectedSocketImpl final : public ConnectedSocketImpl {
   {
     size_t sent = 0;
     while (1) {
+      MSGR_SIGPIPE_STOPPER;
       ssize_t r;
       r = ::sendmsg(fd, &msg, MSG_NOSIGNAL | (more ? MSG_MORE : 0));
       if (r < 0) {
diff --git a/src/msg/simple/Pipe.cc b/src/msg/simple/Pipe.cc
index 368efdb82a53e..848efd45c0090 100644
--- a/src/msg/simple/Pipe.cc
+++ b/src/msg/simple/Pipe.cc
@@ -2259,6 +2259,7 @@ int Pipe::read_message(Message **pm, AuthSessionHandler* auth_handler)
 
 int Pipe::do_sendmsg(struct msghdr *msg, unsigned len, bool more)
 {
+  MSGR_SIGPIPE_STOPPER;
   while (len > 0) {
     int r;
     r = ::sendmsg(sd, msg, MSG_NOSIGNAL | (more ? MSG_MORE : 0));
@@ -2662,6 +2663,7 @@ int Pipe::tcp_write(const char *buf, unsigned len)
   //lgeneric_dout(cct, DBL) << "tcp_write writing " << len << dendl;
   assert(len > 0);
   while (len > 0) {
+    MSGR_SIGPIPE_STOPPER;
     int did = ::send( sd, buf, len, MSG_NOSIGNAL );
     if (did < 0) {
       //lgeneric_dout(cct, 1) << "tcp_write error did = " << did << " " << cpp_strerror(errno) << dendl;
-- 
2.39.5