mds: add dispatch killpoint and delay configs

For testing purposes to slow down the MDS request processing or kill it at
random times.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in
index e8116f275237ba6ef4852cc4205a59623c1d77c6..a9b24ef637d8b19702fb8b80586dfd3132ca0f3a 100644 (file)
--- a/src/common/options/mds.yaml.in
+++ b/src/common/options/mds.yaml.in
@@ -1680,3 +1680,19 @@ options:
   - mds
   flags:
   - runtime
+- name: mds_server_dispatch_killpoint_random
+  type: float
+  level: dev
+  default: 0.0
+  services:
+  - mds
+  flags:
+  - runtime
+- name: mds_server_dispatch_client_request_delay
+  type: millisecs
+  level: dev
+  default: 0
+  services:
+  - mds
+  flags:
+  - runtime

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index c9e50d79fa209d7e855e930dd5d6c2a6d1ae2392..ebc37f4b9db204a620ba5b804d2344658b95fad1 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -4029,6 +4029,8 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const
     "mds_debug_subtrees",
     "mds_dir_max_entries",
     "mds_dump_cache_threshold_file",
+    "mds_server_dispatch_client_request_delay",
+    "mds_server_dispatch_killpoint_random",
     "mds_dump_cache_threshold_formatter",
     "mds_enable_op_tracker",
     "mds_export_ephemeral_distributed",

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index aae51735d9145ea4a9d2e74f9f3a0e1e345c3b91..95f13cc54f3f924827b4f162b3d76872fa3cf270 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -273,6 +273,8 @@ Server::Server(MDSRank *m, MetricsHandler *metrics_handler) :
   caps_throttle_retry_request_timeout = g_conf().get_val<double>("mds_cap_acquisition_throttle_retry_request_timeout");
   dir_max_entries = g_conf().get_val<uint64_t>("mds_dir_max_entries");
   bal_fragment_size_max = g_conf().get_val<int64_t>("mds_bal_fragment_size_max");
+  dispatch_client_request_delay = g_conf().get_val<std::chrono::milliseconds>("mds_server_dispatch_client_request_delay");
+  dispatch_killpoint_random = g_conf().get_val<double>("mds_server_dispatch_killpoint_random");
   supported_features = feature_bitset_t(CEPHFS_FEATURES_MDS_SUPPORTED);
   supported_metric_spec = feature_bitset_t(CEPHFS_METRIC_FEATURES_ALL);
 }
@@ -1375,6 +1377,16 @@ void Server::handle_conf_change(const std::set<std::string>& changed) {
   if (changed.count("mds_inject_rename_corrupt_dentry_first")) {
     inject_rename_corrupt_dentry_first = g_conf().get_val<double>("mds_inject_rename_corrupt_dentry_first");
   }
+  if (changed.count("mds_server_dispatch_client_request_delay")) {
+    dispatch_client_request_delay = g_conf().get_val<std::chrono::milliseconds>("mds_server_dispatch_client_request_delay");
+    dout(20) << __func__ << " mds_server_dispatch_client_request_delay now "
+            << dispatch_client_request_delay << dendl;
+  }
+  if (changed.count("mds_server_dispatch_killpoint_random")) {
+    dispatch_killpoint_random = g_conf().get_val<double>("mds_server_dispatch_killpoint_random");
+    dout(20) << __func__ << " mds_server_dispatch_killpoint_random now "
+            << dispatch_killpoint_random << dendl;
+  }
 }
 
 /*
@@ -2667,6 +2679,14 @@ void Server::dispatch_client_request(const MDRequestRef& mdr)
 
   dout(7) << "dispatch_client_request " << *req << dendl;
 
+  auto zeroms = std::chrono::milliseconds::zero();
+  if (unlikely(dispatch_client_request_delay > zeroms)) {
+    std::this_thread::sleep_for(dispatch_client_request_delay);
+  }
+  if (unlikely(dispatch_killpoint_random > 0.0) && dispatch_killpoint_random >= ceph::util::generate_random_number(0.0, 1.0)) {
+    ceph_abort("dispatch_killpoint_random");
+  }
+
   if (req->may_write() && mdcache->is_readonly()) {
     dout(10) << " read-only FS" << dendl;
     respond_to_request(mdr, -CEPHFS_EROFS);

diff --git a/src/mds/Server.h b/src/mds/Server.h
index 152ab1b701992a52801b1fbf4b227c9478e96926..cbfe5c8cf5dd2cf4778ba9d50d3ad7752e3b2122 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -566,6 +566,9 @@ private:
   double max_caps_throttle_ratio;
   double caps_throttle_retry_request_timeout;
 
+  std::chrono::milliseconds dispatch_client_request_delay{0};
+  double dispatch_killpoint_random{0.0};
+
   size_t alternate_name_max = g_conf().get_val<Option::size_t>("mds_alternate_name_max");
   size_t fscrypt_last_block_max_size = g_conf().get_val<Option::size_t>("mds_fscrypt_last_block_max_size");