rgw: fix up signal handling

OMG libfcgi is annoying with shutdown and signals.  You need to close
the fd *and* resend a signal to ensure that you kick the accept loop
hard enough to make it shut down.

Document this, and switch to the async signal handlers.  Put them
tightly around the runtime loop as we do with other daemons.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index 514e9e47171cdc1c29f328586ce106b81c5bc407..07c0aebfa0a07ef1df77cbae6bd0fa7b08f347fa 100644 (file)
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -65,9 +65,7 @@
 
 using namespace std;
 
-static sighandler_t sighandler_usr1;
 static sighandler_t sighandler_alrm;
-static sighandler_t sighandler_term;
 
 class RGWProcess;
 
@@ -262,14 +260,27 @@ void RGWProcess::run()
   m_tp.stop();
 }
 
-static void godown_handler(int signum)
+static void handle_sigterm(int signum)
 {
+  dout(1) << __func__ << dendl;
   FCGX_ShutdownPending();
+
+  // close the fd, so that accept can't start again.
   pprocess->close_fd();
-  signal(signum, sighandler_usr1);
-  uint64_t secs = g_ceph_context->_conf->rgw_exit_timeout_secs;
-  if (secs)
-    alarm(secs);
+
+  // send a signal to make fcgi's accept(2) wake up.  unfortunately the
+  // initial signal often isn't sufficient because we race with accept's
+  // check of the flag wet by ShutdownPending() above.
+  if (signum != SIGUSR1) {
+    kill(getpid(), SIGUSR1);
+
+    // safety net in case we get stuck doing an orderly shutdown.
+    uint64_t secs = g_ceph_context->_conf->rgw_exit_timeout_secs;
+    if (secs)
+      alarm(secs);
+    dout(1) << __func__ << " set alarm for " << secs << dendl;
+  }
+
 }
 
 static void godown_alarm(int signum)
@@ -483,17 +494,8 @@ int main(int argc, const char **argv)
   
   curl_global_init(CURL_GLOBAL_ALL);
   
-  sighandler_usr1 = signal(SIGUSR1, godown_handler);
-  sighandler_alrm = signal(SIGALRM, godown_alarm);
-  
-  init_async_signal_handler();
-  register_async_signal_handler(SIGHUP, sighup_handler);
-
   FCGX_Init();
 
-  sighandler_term = signal(SIGTERM, godown_alarm);
-  
-
   int r = 0;
   RGWRados *store = RGWStoreManager::get_storage(g_ceph_context, true);
   if (!store) {
@@ -563,7 +565,22 @@ int main(int argc, const char **argv)
 
   pprocess = new RGWProcess(g_ceph_context, store, olog, g_conf->rgw_thread_pool_size, &rest);
 
+  init_async_signal_handler();
+  register_async_signal_handler(SIGHUP, sighup_handler);
+  register_async_signal_handler(SIGTERM, handle_sigterm);
+  register_async_signal_handler(SIGINT, handle_sigterm);
+  register_async_signal_handler(SIGUSR1, handle_sigterm);
+
+  sighandler_alrm = signal(SIGALRM, godown_alarm);
+
   pprocess->run();
+  derr << "shutting down" << dendl;
+
+  unregister_async_signal_handler(SIGHUP, sighup_handler);
+  unregister_async_signal_handler(SIGTERM, handle_sigterm);
+  unregister_async_signal_handler(SIGINT, handle_sigterm);
+  unregister_async_signal_handler(SIGUSR1, handle_sigterm);
+  shutdown_async_signal_handler();
 
   delete pprocess;
 
@@ -577,15 +594,13 @@ int main(int argc, const char **argv)
 
   rgw_perf_stop(g_ceph_context);
 
-  unregister_async_signal_handler(SIGHUP, sighup_handler);
-
   RGWStoreManager::close_storage(store);
 
   rgw_tools_cleanup();
   curl_global_cleanup();
-  g_ceph_context->put();
 
-  shutdown_async_signal_handler();
+  dout(1) << "final shutdown" << dendl;
+  g_ceph_context->put();
 
   ceph::crypto::shutdown();