From fa2aae39716113989129f8e6d626655d8d09da62 Mon Sep 17 00:00:00 2001 From: Rishabh Dave Date: Fri, 8 Mar 2024 21:09:18 +0530 Subject: [PATCH] cephfs,mon: require confirmation to fail unhealthy MDS When running the command "ceph mds fail" for an MDS that is unhealthy due to, MDS_CACHE_OVERSIZED or MDS_TRIM, user must pass confirmation flag. Else, the command will fail and print an appropriate error message. Restarting an MDS with such health warnings is not recommended since it will have a slow reocvery during restart which will create new problems. Fixes: https://tracker.ceph.com/issues/61866 Signed-off-by: Rishabh Dave (cherry picked from commit eeda00eea5043d3ba806695a207b732cb53b35c4) --- src/mon/FSCommands.h | 8 ++++++++ src/mon/MDSMonitor.cc | 25 +++++++++++++++++++++++++ src/mon/MDSMonitor.h | 2 ++ src/mon/MonCommands.h | 3 ++- 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/mon/FSCommands.h b/src/mon/FSCommands.h index a8714129693c..79609c335446 100644 --- a/src/mon/FSCommands.h +++ b/src/mon/FSCommands.h @@ -91,4 +91,12 @@ public: std::ostream &ss) = 0; }; + +static constexpr auto errmsg_for_unhealthy_mds = \ + "MDS has one of two health warnings which could extend recovery: " + "MDS_TRIM or MDS_CACHE_OVERSIZED. MDS failover is not recommended " + "since it might cause unexpected file system unavailability. If " + "you wish to proceed, pass --yes-i-really-mean-it"; + + #endif diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc index 88894d73fc88..c619eb79c3f3 100644 --- a/src/mon/MDSMonitor.cc +++ b/src/mon/MDSMonitor.cc @@ -1460,6 +1460,23 @@ out: } } +bool MDSMonitor::has_health_warnings(vector warnings) +{ + for (auto& [gid, health] : pending_daemon_health) { + for (auto& metric : health.metrics) { + // metric.type here is the type of health warning. We are only + // looking for types of health warnings passed to this func member + // through variable "warnings". + auto it = std::find(warnings.begin(), warnings.end(), metric.type); + if (it != warnings.end()) { + return true; + } + } + } + + return false; +} + int MDSMonitor::filesystem_command( FSMap &fsmap, MonOpRequestRef op, @@ -1497,6 +1514,8 @@ int MDSMonitor::filesystem_command( } else if (prefix == "mds fail") { string who; cmd_getval(cmdmap, "role_or_gid", who); + bool confirm = false; + cmd_getval(cmdmap, "yes_i_really_mean_it", confirm); MDSMap::mds_info_t failed_info; mds_gid_t gid = gid_from_arg(fsmap, who, ss); @@ -1516,6 +1535,12 @@ int MDSMonitor::filesystem_command( return -EPERM; } + if (!confirm && + has_health_warnings({MDS_HEALTH_TRIM, MDS_HEALTH_CACHE_OVERSIZED})) { + ss << errmsg_for_unhealthy_mds; + return -EPERM; + } + r = fail_mds(fsmap, ss, who, &failed_info); if (r < 0 && r == -EAGAIN) { mon.osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op)); diff --git a/src/mon/MDSMonitor.h b/src/mon/MDSMonitor.h index c41246992d5c..0157a47177c0 100644 --- a/src/mon/MDSMonitor.h +++ b/src/mon/MDSMonitor.h @@ -20,6 +20,7 @@ #include #include +#include #include "include/types.h" #include "PaxosFSMap.h" @@ -51,6 +52,7 @@ class MDSMonitor : public PaxosService, public PaxosFSMap, protected CommandHand bool preprocess_query(MonOpRequestRef op) override; // true if processed. bool prepare_update(MonOpRequestRef op) override; bool should_propose(double& delay) override; + bool has_health_warnings(std::vector warnings); bool should_print_status() const { auto& fs = get_fsmap(); diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index cb412c1494c0..a6bb07132a6e 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -319,7 +319,8 @@ COMMAND_WITH_FLAG("mds set_state " "name=gid,type=CephInt,range=0 " "name=state,type=CephInt,range=0|20", "set mds state of to ", "mds", "rw", FLAG(HIDDEN)) -COMMAND("mds fail name=role_or_gid,type=CephString", +COMMAND("mds fail name=role_or_gid,type=CephString " + "name=yes_i_really_mean_it,type=CephBool,req=false", "Mark MDS failed: trigger a failover if a standby is available", "mds", "rw") COMMAND("mds repaired name=role,type=CephString", -- 2.47.3