mds: optionally forbid to use standby for another fs as last resort

Signed-off-by: Mykola Golub <mykola.golub@clyso.com>

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index dabb3f1b6c2715599236dc863f7059033da76499..11c3692c115fd043f3b4116f24525914610626af 100644 (file)
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -223,3 +223,8 @@ Relevant tracker: https://tracker.ceph.com/issues/57090
   key/value set for a filesystem extended attributes.  It effectively replaces
   the old per-MDS `max_xattr_pairs_size` setting, which is now dropped.
   Relevant tracker: https://tracker.ceph.com/issues/55725
+
+* Introduced a new file system flag `refuse_standby_for_another_fs` that can be
+set using the `fs set` command. This flag prevents using a standby for another
+file system (join_fs = X) when standby for the current filesystem is not available.
+Relevant tracker: https://tracker.ceph.com/issues/61599

diff --git a/doc/cephfs/standby.rst b/doc/cephfs/standby.rst
index 70c981301c221e37364daebca2d28dc134470142..a64c3e41c8abe42d08d6b38cf44a7fa9b43d5fa3 100644 (file)
--- a/doc/cephfs/standby.rst
+++ b/doc/cephfs/standby.rst
@@ -118,10 +118,16 @@ enforces this affinity.
 When failing over MDS daemons, a cluster's monitors will prefer standby daemons with
 ``mds_join_fs`` equal to the file system ``name`` with the failed ``rank``.  If no
 standby exists with ``mds_join_fs`` equal to the file system ``name``, it will
-choose an unqualified standby (no setting for ``mds_join_fs``) for the replacement,
-or any other available standby, as a last resort. Note, this does not change the
-behavior that ``standby-replay`` daemons are always selected before
-other standbys.
+choose an unqualified standby (no setting for ``mds_join_fs``) for the replacement.
+As a last resort, a standby for another filesystem will be chosen, although this
+behavior can be disabled:
+
+::
+
+    ceph fs set <fs name> refuse_standby_for_another_fs true
+
+Note, configuring MDS file system affinity does not change the behavior that
+``standby-replay`` daemons are always selected before other standbys.
 
 Even further, the monitors will regularly examine the CephFS file systems even when
 stable to check if a standby with stronger affinity is available to replace an

diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index 42e5e53b438a39c62e2581d0b8387adbe8bd2569..7fd5316804d35873a3770f81a08e181b46fd5e9b 100644 (file)
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -290,6 +290,8 @@ struct ceph_mon_subscribe_ack {
 #define CEPH_MDSMAP_ALLOW_STANDBY_REPLAY         (1<<5)  /* cluster alllowed to enable MULTIMDS */
 #define CEPH_MDSMAP_REFUSE_CLIENT_SESSION        (1<<6)  /* cluster allowed to refuse client session
                                                             request */
+#define CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS (1<<7) /* fs is forbidden to use standby
+                                                            for another fs */
 #define CEPH_MDSMAP_DEFAULTS (CEPH_MDSMAP_ALLOW_SNAPS | \
                              CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS)
 

diff --git a/src/mds/FSMap.cc b/src/mds/FSMap.cc
index b9ae05ac0f62246831f30ed0d7ed6c9d0df8b1ae..e1c98be1b82e6cb999bf7a3ab168e75b79024fdf 100644 (file)
--- a/src/mds/FSMap.cc
+++ b/src/mds/FSMap.cc
@@ -792,7 +792,8 @@ const MDSMap::mds_info_t* FSMap::get_available_standby(const Filesystem& fs) con
       break;
     } else if (info.join_fscid == FS_CLUSTER_ID_NONE) {
       who = &info; /* vanilla standby */
-    } else if (who == nullptr) {
+    } else if (who == nullptr &&
+              !fs.mds_map.test_flag(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS)) {
       who = &info; /* standby for another fs, last resort */
     }
   }

diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc
index 6ea1de533de7ce9d42e6e69b75bd7ecd5cca3cab..f1613dbf323ae9c472a6ee7db6662e08bff68715 100644 (file)
--- a/src/mds/MDSMap.cc
+++ b/src/mds/MDSMap.cc
@@ -236,6 +236,7 @@ void MDSMap::dump_flags_state(Formatter *f) const
     f->dump_bool(flag_display.at(CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS), allows_multimds_snaps());
     f->dump_bool(flag_display.at(CEPH_MDSMAP_ALLOW_STANDBY_REPLAY), allows_standby_replay());
     f->dump_bool(flag_display.at(CEPH_MDSMAP_REFUSE_CLIENT_SESSION), test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION));
+    f->dump_bool(flag_display.at(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS), test_flag(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS));
     f->close_section();
 }
 
@@ -378,6 +379,8 @@ void MDSMap::print_flags(std::ostream& out) const {
     out << " " << flag_display.at(CEPH_MDSMAP_ALLOW_STANDBY_REPLAY);
   if (test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION))
     out << " " << flag_display.at(CEPH_MDSMAP_REFUSE_CLIENT_SESSION);
+  if (test_flag(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS))
+    out << " " << flag_display.at(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS);
 }
 
 void MDSMap::get_health(list<pair<health_status_t,string> >& summary,

diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h
index 7e1814e5977a49e7b71b40537badd2af73665f62..52bc0fa367f087836614c739476cd9d9ba453e5b 100644 (file)
--- a/src/mds/MDSMap.h
+++ b/src/mds/MDSMap.h
@@ -675,7 +675,8 @@ private:
     {CEPH_MDSMAP_ALLOW_SNAPS, "allow_snaps"},
     {CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS, "allow_multimds_snaps"},
     {CEPH_MDSMAP_ALLOW_STANDBY_REPLAY, "allow_standby_replay"},
-    {CEPH_MDSMAP_REFUSE_CLIENT_SESSION, "refuse_client_session"}
+    {CEPH_MDSMAP_REFUSE_CLIENT_SESSION, "refuse_client_session"},
+    {CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS, "refuse_standby_for_another_fs"}
   };
 };
 WRITE_CLASS_ENCODER_FEATURES(MDSMap::mds_info_t)

diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc
index 4ea716acaf50dfdac554bcd558e047a92f25b5bc..0983d435f518c3a92dfd53e4e137df4cff99220e 100644 (file)
--- a/src/mon/FSCommands.cc
+++ b/src/mon/FSCommands.cc
@@ -720,6 +720,38 @@ public:
             ss << "client(s) already allowed to establish new session(s)";
           }
       }
+    } else if (var == "refuse_standby_for_another_fs") {
+      bool refuse_standby_for_another_fs = false;
+      int r = parse_bool(val, &refuse_standby_for_another_fs, ss);
+      if (r != 0) {
+        return r;
+      }
+
+      if (refuse_standby_for_another_fs) {
+        if (!(fs->mds_map.test_flag(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS))) {
+          fsmap.modify_filesystem(
+            fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+          {
+            fs->mds_map.set_flag(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS);
+          });
+          ss << "set to refuse standby for another fs";
+        } else {
+          ss << "to refuse standby for another fs is already set";
+        }
+      } else {
+          if (fs->mds_map.test_flag(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS)) {
+            fsmap.modify_filesystem(
+              fs->fscid,
+              [](std::shared_ptr<Filesystem> fs)
+            {
+              fs->mds_map.clear_flag(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS);
+            });
+            ss << "allowed to use standby for another fs";
+          } else {
+            ss << "to use standby for another fs is already allowed";
+          }
+      }
     } else {
       ss << "unknown variable " << var;
       return -EINVAL;

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index ec319c68ee4ec3db3d07fb55920ef42fc59fb2b3..05fda59b1735b55976d844fc43652682e1ebbce6 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -378,7 +378,7 @@ COMMAND("fs set "
         "|allow_new_snaps|inline_data|cluster_down|allow_dirfrags|balancer"
         "|standby_count_wanted|session_timeout|session_autoclose"
         "|allow_standby_replay|down|joinable|min_compat_client|bal_rank_mask"
-       "|refuse_client_session|max_xattr_size "
+       "|refuse_client_session|max_xattr_size|refuse_standby_for_another_fs "
        "name=val,type=CephString "
        "name=yes_i_really_mean_it,type=CephBool,req=false "
        "name=yes_i_really_really_mean_it,type=CephBool,req=false",