osd: add osd_fast_shutdown option (default true)

If we get a SIGINT or SIGTERM or are deleted from the OSDMap, do a fast
shutdown by exiting immediately.  This has a few important benefits:

 - We immediately stop responding (binding) to any sockets, which means
   other OSDs will immediately decide we are down (and dead!).  This
   minimizes IO interruption.
 - We avoid the complex "clean" shutdown process, which is historically a
   source of bugs.

In reality, the only purpose of the "clean" shutdown is to try to tear down
everything in memory so we can do memory leak checking with valgrind.  Set
this option to false for valgrind QA runs so we can still do that.

Not that with the new read leases in octopus, we rely on the default
behavior that a ECONNREFUSED is taken to mean that the OSD is fully dead,
so that we don't have to wait for any leases to time out.  This works in
sane environments with normal IP networks, but that behavior could
conceivably be a bad idea if there are some weird network shenanigans
going on.  If osd_fast_fail_on_connection_refused were disabled, then this
fast shutdown procedure might be *worse* than the clean shutdown because
we would have to wait for the heartbeat timeout.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/qa/suites/fs/verify/validater/valgrind.yaml b/qa/suites/fs/verify/validater/valgrind.yaml
index fc4c459d1ac2e481e5999d832047cd36c3475ffa..a5c081542d8093c332ef363b8537abd090d2dc8a 100644 (file)
--- a/qa/suites/fs/verify/validater/valgrind.yaml
+++ b/qa/suites/fs/verify/validater/valgrind.yaml
@@ -17,6 +17,8 @@ overrides:
         mds heartbeat grace: 60
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     valgrind:
       mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes]
       osd: [--tool=memcheck]

diff --git a/qa/suites/rados/singleton-flat/valgrind-leaks.yaml b/qa/suites/rados/singleton-flat/valgrind-leaks.yaml
index e70a5e40a3f6ffd6ad04d59d067a0df443cc2233..c41f75fce02a9506ad3d9abb3f9bf9a5ef24bf71 100644 (file)
--- a/qa/suites/rados/singleton-flat/valgrind-leaks.yaml
+++ b/qa/suites/rados/singleton-flat/valgrind-leaks.yaml
@@ -23,6 +23,8 @@ overrides:
         osd max object namespace len: 64
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     valgrind:
       mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes]
       osd: [--tool=memcheck]

diff --git a/qa/suites/rados/verify/validater/valgrind.yaml b/qa/suites/rados/verify/validater/valgrind.yaml
index 8cceca5957e92ee4ac81e9d29e2e6232aec43838..fe8e0e1cac04575c4b5fedd2304b4b090f000ba8 100644 (file)
--- a/qa/suites/rados/verify/validater/valgrind.yaml
+++ b/qa/suites/rados/verify/validater/valgrind.yaml
@@ -13,6 +13,8 @@ overrides:
         debug refs: 5
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     log-whitelist:
       - overall HEALTH_
 # valgrind is slow.. we might get PGs stuck peering etc

diff --git a/qa/suites/rgw/multisite/valgrind.yaml b/qa/suites/rgw/multisite/valgrind.yaml
index 08fad9da02381e15955f85b1a236f55d70455a75..99489951b4a9bca80796f424522716fa346f3053 100644 (file)
--- a/qa/suites/rgw/multisite/valgrind.yaml
+++ b/qa/suites/rgw/multisite/valgrind.yaml
@@ -11,6 +11,8 @@ overrides:
         osd heartbeat grace: 40
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     valgrind:
       mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes]
       osd: [--tool=memcheck]

diff --git a/qa/suites/rgw/verify/validater/valgrind.yaml b/qa/suites/rgw/verify/validater/valgrind.yaml
index 5397cd9142a1c04011275e3a32d2879b62a39f3e..442bba879fa5a63eb6f944de2757625563ce8963 100644 (file)
--- a/qa/suites/rgw/verify/validater/valgrind.yaml
+++ b/qa/suites/rgw/verify/validater/valgrind.yaml
@@ -12,6 +12,8 @@ overrides:
         osd heartbeat grace: 40
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     valgrind:
       mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes]
       osd: [--tool=memcheck]

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index 01db8d78ce4034f7e701d7ba9e586eb64ab59101..5685011eda8634f638be604186f165139513bbdb 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -764,6 +764,7 @@ OPTION(osd_op_history_slow_op_size, OPT_U32)           // Max number of slow ops
 OPTION(osd_op_history_slow_op_threshold, OPT_DOUBLE) // track the op if over this threshold
 OPTION(osd_target_transaction_size, OPT_INT)     // to adjust various transactions that batch smaller items
 OPTION(osd_failsafe_full_ratio, OPT_FLOAT) // what % full makes an OSD "full" (failsafe)
+OPTION(osd_fast_shutdown, OPT_BOOL)
 OPTION(osd_fast_fail_on_connection_refused, OPT_BOOL) // immediately mark OSDs as down once they refuse to accept connections
 
 OPTION(osd_pg_object_context_cache_count, OPT_INT)

diff --git a/src/common/options.cc b/src/common/options.cc
index 34406ed6cb6b5e1363d7f377b279d4bd930d24a2..3e5c30d53c3cb81fac0a3569d2369cd48b40106b 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -3498,6 +3498,11 @@ std::vector<Option> get_global_options() {
     .set_default(.97)
     .set_description(""),
 
+    Option("osd_fast_shutdown", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_description("Fast, immediate shutdown")
+    .set_long_description("Setting this to false makes the OSD do a slower teardown of all state when it receives a SIGINT or SIGTERM or when shutting down for any other reason.  That slow shutdown is primarilyy useful for doing memory leak checking with valgrind."),
+
     Option("osd_fast_fail_on_connection_refused", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
     .set_description(""),

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 6b592ff85bfa314d216a16fe762fd976c52bb00e..37b2a53cc38adc84d24b59ec8742104e68b81ca3 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -4016,6 +4016,12 @@ void OSD::create_recoverystate_perf()
 
 int OSD::shutdown()
 {
+  if (cct->_conf->osd_fast_shutdown) {
+    derr << "*** Immediate shutdown (osd_fast_shutdown=true) ***" << dendl;
+    cct->_log->flush();
+    _exit(0);
+  }
+
   if (!service.prepare_to_stop())
     return 0; // already shutting down
   osd_lock.lock();

diff --git a/src/vstart.sh b/src/vstart.sh
index 9b83a0525f4ac9bcf58e82648f47fa9bcb7568d2..a9c8bc4e14f027ac70c64981fde8e23a8b8c6526 100755 (executable)
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -710,6 +710,7 @@ $DAEMONOPTS
         osd class dir = $OBJCLASS_PATH
         osd class load list = *
         osd class default list = *
+        osd fast shutdown = false
 
         filestore wbthrottle xfs ios start flusher = 10
         filestore wbthrottle xfs ios hard limit = 20