From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 28 Jul 2022 18:35:47 +0000 (-0400)
Subject: mds: add dispatch killpoint and delay configs
X-Git-Tag: testing/wip-pdonnell-testing-20240411.210829-debug^2
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=007008121ac99d4c560fe4ac0420d1b3cd00e678;p=ceph-ci.git

mds: add dispatch killpoint and delay configs

For testing purposes to slow down the MDS request processing or kill it at
random times.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---

diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in
index e8116f27523..a9b24ef637d 100644
--- a/src/common/options/mds.yaml.in
+++ b/src/common/options/mds.yaml.in
@@ -1680,3 +1680,19 @@ options:
   - mds
   flags:
   - runtime
+- name: mds_server_dispatch_killpoint_random
+  type: float
+  level: dev
+  default: 0.0
+  services:
+  - mds
+  flags:
+  - runtime
+- name: mds_server_dispatch_client_request_delay
+  type: millisecs
+  level: dev
+  default: 0
+  services:
+  - mds
+  flags:
+  - runtime
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index c9e50d79fa2..ebc37f4b9db 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -4029,6 +4029,8 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const
     "mds_debug_subtrees",
     "mds_dir_max_entries",
     "mds_dump_cache_threshold_file",
+    "mds_server_dispatch_client_request_delay",
+    "mds_server_dispatch_killpoint_random",
     "mds_dump_cache_threshold_formatter",
     "mds_enable_op_tracker",
     "mds_export_ephemeral_distributed",
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index aae51735d91..95f13cc54f3 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -273,6 +273,8 @@ Server::Server(MDSRank *m, MetricsHandler *metrics_handler) :
   caps_throttle_retry_request_timeout = g_conf().get_val<double>("mds_cap_acquisition_throttle_retry_request_timeout");
   dir_max_entries = g_conf().get_val<uint64_t>("mds_dir_max_entries");
   bal_fragment_size_max = g_conf().get_val<int64_t>("mds_bal_fragment_size_max");
+  dispatch_client_request_delay = g_conf().get_val<std::chrono::milliseconds>("mds_server_dispatch_client_request_delay");
+  dispatch_killpoint_random = g_conf().get_val<double>("mds_server_dispatch_killpoint_random");
   supported_features = feature_bitset_t(CEPHFS_FEATURES_MDS_SUPPORTED);
   supported_metric_spec = feature_bitset_t(CEPHFS_METRIC_FEATURES_ALL);
 }
@@ -1375,6 +1377,16 @@ void Server::handle_conf_change(const std::set<std::string>& changed) {
   if (changed.count("mds_inject_rename_corrupt_dentry_first")) {
     inject_rename_corrupt_dentry_first = g_conf().get_val<double>("mds_inject_rename_corrupt_dentry_first");
   }
+  if (changed.count("mds_server_dispatch_client_request_delay")) {
+    dispatch_client_request_delay = g_conf().get_val<std::chrono::milliseconds>("mds_server_dispatch_client_request_delay");
+    dout(20) << __func__ << " mds_server_dispatch_client_request_delay now "
+            << dispatch_client_request_delay << dendl;
+  }
+  if (changed.count("mds_server_dispatch_killpoint_random")) {
+    dispatch_killpoint_random = g_conf().get_val<double>("mds_server_dispatch_killpoint_random");
+    dout(20) << __func__ << " mds_server_dispatch_killpoint_random now "
+            << dispatch_killpoint_random << dendl;
+  }
 }
 
 /*
@@ -2667,6 +2679,14 @@ void Server::dispatch_client_request(const MDRequestRef& mdr)
 
   dout(7) << "dispatch_client_request " << *req << dendl;
 
+  auto zeroms = std::chrono::milliseconds::zero();
+  if (unlikely(dispatch_client_request_delay > zeroms)) {
+    std::this_thread::sleep_for(dispatch_client_request_delay);
+  }
+  if (unlikely(dispatch_killpoint_random > 0.0) && dispatch_killpoint_random >= ceph::util::generate_random_number(0.0, 1.0)) {
+    ceph_abort("dispatch_killpoint_random");
+  }
+
   if (req->may_write() && mdcache->is_readonly()) {
     dout(10) << " read-only FS" << dendl;
     respond_to_request(mdr, -CEPHFS_EROFS);
diff --git a/src/mds/Server.h b/src/mds/Server.h
index 152ab1b7019..cbfe5c8cf5d 100644
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -566,6 +566,9 @@ private:
   double max_caps_throttle_ratio;
   double caps_throttle_retry_request_timeout;
 
+  std::chrono::milliseconds dispatch_client_request_delay{0};
+  double dispatch_killpoint_random{0.0};
+
   size_t alternate_name_max = g_conf().get_val<Option::size_t>("mds_alternate_name_max");
   size_t fscrypt_last_block_max_size = g_conf().get_val<Option::size_t>("mds_fscrypt_last_block_max_size");