From cf352c3ac0bd87d8b7e0c52ac724f94576ae5aa7 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 15 Nov 2019 09:31:50 -0600
Subject: [PATCH] osd: add osd_fast_shutdown option (default true)

If we get a SIGINT or SIGTERM or are deleted from the OSDMap, do a fast
shutdown by exiting immediately.  This has a few important benefits:

 - We immediately stop responding (binding) to any sockets, which means
   other OSDs will immediately decide we are down (and dead!).  This
   minimizes IO interruption.
 - We avoid the complex "clean" shutdown process, which is historically a
   source of bugs.

In reality, the only purpose of the "clean" shutdown is to try to tear down
everything in memory so we can do memory leak checking with valgrind.  Set
this option to false for valgrind QA runs so we can still do that.

Not that with the new read leases in octopus, we rely on the default
behavior that a ECONNREFUSED is taken to mean that the OSD is fully dead,
so that we don't have to wait for any leases to time out.  This works in
sane environments with normal IP networks, but that behavior could
conceivably be a bad idea if there are some weird network shenanigans
going on.  If osd_fast_fail_on_connection_refused were disabled, then this
fast shutdown procedure might be *worse* than the clean shutdown because
we would have to wait for the heartbeat timeout.

Signed-off-by: Sage Weil <sage@redhat.com>
---
 qa/suites/fs/verify/validater/valgrind.yaml        | 2 ++
 qa/suites/rados/singleton-flat/valgrind-leaks.yaml | 2 ++
 qa/suites/rados/verify/validater/valgrind.yaml     | 2 ++
 qa/suites/rgw/multisite/valgrind.yaml              | 2 ++
 qa/suites/rgw/verify/validater/valgrind.yaml       | 2 ++
 src/common/legacy_config_opts.h                    | 1 +
 src/common/options.cc                              | 5 +++++
 src/osd/OSD.cc                                     | 6 ++++++
 src/vstart.sh                                      | 1 +
 9 files changed, 23 insertions(+)

diff --git a/qa/suites/fs/verify/validater/valgrind.yaml b/qa/suites/fs/verify/validater/valgrind.yaml
index fc4c459d1ac..a5c081542d8 100644
--- a/qa/suites/fs/verify/validater/valgrind.yaml
+++ b/qa/suites/fs/verify/validater/valgrind.yaml
@@ -17,6 +17,8 @@ overrides:
         mds heartbeat grace: 60
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     valgrind:
       mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes]
       osd: [--tool=memcheck]
diff --git a/qa/suites/rados/singleton-flat/valgrind-leaks.yaml b/qa/suites/rados/singleton-flat/valgrind-leaks.yaml
index e70a5e40a3f..c41f75fce02 100644
--- a/qa/suites/rados/singleton-flat/valgrind-leaks.yaml
+++ b/qa/suites/rados/singleton-flat/valgrind-leaks.yaml
@@ -23,6 +23,8 @@ overrides:
         osd max object namespace len: 64
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     valgrind:
       mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes]
       osd: [--tool=memcheck]
diff --git a/qa/suites/rados/verify/validater/valgrind.yaml b/qa/suites/rados/verify/validater/valgrind.yaml
index 8cceca5957e..fe8e0e1cac0 100644
--- a/qa/suites/rados/verify/validater/valgrind.yaml
+++ b/qa/suites/rados/verify/validater/valgrind.yaml
@@ -13,6 +13,8 @@ overrides:
         debug refs: 5
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     log-whitelist:
       - overall HEALTH_
 # valgrind is slow.. we might get PGs stuck peering etc
diff --git a/qa/suites/rgw/multisite/valgrind.yaml b/qa/suites/rgw/multisite/valgrind.yaml
index 08fad9da023..99489951b4a 100644
--- a/qa/suites/rgw/multisite/valgrind.yaml
+++ b/qa/suites/rgw/multisite/valgrind.yaml
@@ -11,6 +11,8 @@ overrides:
         osd heartbeat grace: 40
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     valgrind:
       mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes]
       osd: [--tool=memcheck]
diff --git a/qa/suites/rgw/verify/validater/valgrind.yaml b/qa/suites/rgw/verify/validater/valgrind.yaml
index 5397cd9142a..442bba879fa 100644
--- a/qa/suites/rgw/verify/validater/valgrind.yaml
+++ b/qa/suites/rgw/verify/validater/valgrind.yaml
@@ -12,6 +12,8 @@ overrides:
         osd heartbeat grace: 40
       mon:
         mon osd crush smoke test: false
+      osd:
+        osd fast shutdown: false
     valgrind:
       mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes]
       osd: [--tool=memcheck]
diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index 01db8d78ce4..5685011eda8 100644
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -764,6 +764,7 @@ OPTION(osd_op_history_slow_op_size, OPT_U32)           // Max number of slow ops
 OPTION(osd_op_history_slow_op_threshold, OPT_DOUBLE) // track the op if over this threshold
 OPTION(osd_target_transaction_size, OPT_INT)     // to adjust various transactions that batch smaller items
 OPTION(osd_failsafe_full_ratio, OPT_FLOAT) // what % full makes an OSD "full" (failsafe)
+OPTION(osd_fast_shutdown, OPT_BOOL)
 OPTION(osd_fast_fail_on_connection_refused, OPT_BOOL) // immediately mark OSDs as down once they refuse to accept connections
 
 OPTION(osd_pg_object_context_cache_count, OPT_INT)
diff --git a/src/common/options.cc b/src/common/options.cc
index 34406ed6cb6..3e5c30d53c3 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -3498,6 +3498,11 @@ std::vector<Option> get_global_options() {
     .set_default(.97)
     .set_description(""),
 
+    Option("osd_fast_shutdown", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_description("Fast, immediate shutdown")
+    .set_long_description("Setting this to false makes the OSD do a slower teardown of all state when it receives a SIGINT or SIGTERM or when shutting down for any other reason.  That slow shutdown is primarilyy useful for doing memory leak checking with valgrind."),
+
     Option("osd_fast_fail_on_connection_refused", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
     .set_description(""),
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 6b592ff85bf..37b2a53cc38 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -4016,6 +4016,12 @@ void OSD::create_recoverystate_perf()
 
 int OSD::shutdown()
 {
+  if (cct->_conf->osd_fast_shutdown) {
+    derr << "*** Immediate shutdown (osd_fast_shutdown=true) ***" << dendl;
+    cct->_log->flush();
+    _exit(0);
+  }
+
   if (!service.prepare_to_stop())
     return 0; // already shutting down
   osd_lock.lock();
diff --git a/src/vstart.sh b/src/vstart.sh
index 9b83a0525f4..a9c8bc4e14f 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -710,6 +710,7 @@ $DAEMONOPTS
         osd class dir = $OBJCLASS_PATH
         osd class load list = *
         osd class default list = *
+        osd fast shutdown = false
 
         filestore wbthrottle xfs ios start flusher = 10
         filestore wbthrottle xfs ios hard limit = 20
-- 
2.39.5