osd: add osd_fast_shutdown option (default true)

If we get a SIGINT or SIGTERM or are deleted from the OSDMap, do a fast
shutdown by exiting immediately.  This has a few important benefits:

 - We immediately stop responding (binding) to any sockets, which means
   other OSDs will immediately decide we are down (and dead!).  This
   minimizes IO interruption.
 - We avoid the complex "clean" shutdown process, which is historically a
   source of bugs.

In reality, the only purpose of the "clean" shutdown is to try to tear down
everything in memory so we can do memory leak checking with valgrind.  Set
this option to false for valgrind QA runs so we can still do that.

Not that with the new read leases in octopus, we rely on the default
behavior that a ECONNREFUSED is taken to mean that the OSD is fully dead,
so that we don't have to wait for any leases to time out.  This works in
sane environments with normal IP networks, but that behavior could
conceivably be a bad idea if there are some weird network shenanigans
going on.  If osd_fast_fail_on_connection_refused were disabled, then this
fast shutdown procedure might be *worse* than the clean shutdown because
we would have to wait for the heartbeat timeout.

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit cf352c3ac0bd87d8b7e0c52ac724f94576ae5aa7)

diff --git a/qa/suites/fs/verify/validater/valgrind.yaml b/qa/suites/fs/verify/validater/valgrind.yaml
index fc4c459d1ac2e481e5999d832047cd36c3475ffa..a5c081542d8093c332ef363b8537abd090d2dc8a 100644 (file)
--- a/qa/suites/fs/verify/validater/valgrind.yaml
+++ b/qa/suites/fs/verify/validater/valgrind.yaml
@@ -17,6 +17,8 @@ overrides:
         mds heartbeat grace: 60
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     valgrind:
       mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes]
       osd: [--tool=memcheck]

diff --git a/qa/suites/rados/singleton-flat/valgrind-leaks.yaml b/qa/suites/rados/singleton-flat/valgrind-leaks.yaml
index e70a5e40a3f6ffd6ad04d59d067a0df443cc2233..c41f75fce02a9506ad3d9abb3f9bf9a5ef24bf71 100644 (file)
--- a/qa/suites/rados/singleton-flat/valgrind-leaks.yaml
+++ b/qa/suites/rados/singleton-flat/valgrind-leaks.yaml
@@ -23,6 +23,8 @@ overrides:
         osd max object namespace len: 64
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     valgrind:
       mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes]
       osd: [--tool=memcheck]

diff --git a/qa/suites/rados/verify/validater/valgrind.yaml b/qa/suites/rados/verify/validater/valgrind.yaml
index 8b907c25688f6635941105becc2842f01a50d2dc..2ed6637777f1e95725bffeda6526cf4386c6f746 100644 (file)
--- a/qa/suites/rados/verify/validater/valgrind.yaml
+++ b/qa/suites/rados/verify/validater/valgrind.yaml
@@ -13,6 +13,8 @@ overrides:
         debug refs: 5
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     log-whitelist:
       - overall HEALTH_
 # valgrind is slow.. we might get PGs stuck peering etc

diff --git a/qa/suites/rgw/multisite/valgrind.yaml b/qa/suites/rgw/multisite/valgrind.yaml
index 08fad9da02381e15955f85b1a236f55d70455a75..99489951b4a9bca80796f424522716fa346f3053 100644 (file)
--- a/qa/suites/rgw/multisite/valgrind.yaml
+++ b/qa/suites/rgw/multisite/valgrind.yaml
@@ -11,6 +11,8 @@ overrides:
         osd heartbeat grace: 40
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     valgrind:
       mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes]
       osd: [--tool=memcheck]

diff --git a/qa/suites/rgw/verify/validater/valgrind.yaml b/qa/suites/rgw/verify/validater/valgrind.yaml
index 66571d34d7df374db0672fb127b8b6d2e32884db..4010ccf28b7f515add45b44d2bba418961db8c4e 100644 (file)
--- a/qa/suites/rgw/verify/validater/valgrind.yaml
+++ b/qa/suites/rgw/verify/validater/valgrind.yaml
@@ -12,6 +12,8 @@ overrides:
         osd heartbeat grace: 40
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     valgrind:
       mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes]
       osd: [--tool=memcheck]

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index 79d9c1fa73782ada588fe2705d388fb2f10373ba..7aae31e8eabfe73031dd51545c384610a3f3b681 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -795,6 +795,7 @@ OPTION(osd_op_history_slow_op_size, OPT_U32)           // Max number of slow ops
 OPTION(osd_op_history_slow_op_threshold, OPT_DOUBLE) // track the op if over this threshold
 OPTION(osd_target_transaction_size, OPT_INT)     // to adjust various transactions that batch smaller items
 OPTION(osd_failsafe_full_ratio, OPT_FLOAT) // what % full makes an OSD "full" (failsafe)
+OPTION(osd_fast_shutdown, OPT_BOOL)
 OPTION(osd_fast_fail_on_connection_refused, OPT_BOOL) // immediately mark OSDs as down once they refuse to accept connections
 
 OPTION(osd_pg_object_context_cache_count, OPT_INT)

diff --git a/src/common/options.cc b/src/common/options.cc
index 6d82ed662b0d14d388bc865f1a4614519ce28a84..93e65fd2d67867e814a9a3eeed205ca93eb7121b 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -3864,6 +3864,11 @@ std::vector<Option> get_global_options() {
     .set_default(.97)
     .set_description(""),
 
+    Option("osd_fast_shutdown", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_description("Fast, immediate shutdown")
+    .set_long_description("Setting this to false makes the OSD do a slower teardown of all state when it receives a SIGINT or SIGTERM or when shutting down for any other reason.  That slow shutdown is primarilyy useful for doing memory leak checking with valgrind."),
+
     Option("osd_fast_fail_on_connection_refused", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
     .set_description(""),

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 35c01dc56999c001d7d7ebd0a917bc72502ad6fd..399f095f894319ebd052dafd65387aad559aea56 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -3983,6 +3983,12 @@ void OSD::create_recoverystate_perf()
 
 int OSD::shutdown()
 {
+  if (cct->_conf->osd_fast_shutdown) {
+    derr << "*** Immediate shutdown (osd_fast_shutdown=true) ***" << dendl;
+    cct->_log->flush();
+    _exit(0);
+  }
+
   if (!service.prepare_to_stop())
     return 0; // already shutting down
   osd_lock.Lock();

diff --git a/src/vstart.sh b/src/vstart.sh
index 6d85af72162345b099afb31033362415cd075ded..37aa28b7375329c29e530b98df05e6c3ad61f11b 100755 (executable)
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -609,6 +609,7 @@ $DAEMONOPTS
         osd class dir = $OBJCLASS_PATH
         osd class load list = *
         osd class default list = *
+        osd fast shutdown = false
 
         filestore wbthrottle xfs ios start flusher = 10
         filestore wbthrottle xfs ios hard limit = 20