From 79a6d9d4efc73fe5a722248c62f2bc2fd40559ff Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@ibm.com>
Date: Thu, 29 May 2025 10:13:40 -0400
Subject: [PATCH] mon: cycle through keyring or key_server for auth with mons

After commit `mon: use key_server for looking up mon key`, the mons will now
use the key_server to lookup the `mon.` key when a mon connects.  We need to
make the mons prefer using that key with authenticating during probing other
mons. However, the protocol doesn't allow falling back to another key. This is
necessary if what's in the key_server database is out-of-date due to an earlier
loss of quorum. In that case, the operator should update the local keyring file
and the mon should give that a try if auth fails.

Signed-off-by: Patrick Donnelly <pdonnell@ibm.com>
(cherry picked from commit 827e87d99c9c111574ca05dbcd865e7c9cc98205)
---
 src/mon/Monitor.cc | 33 ++++++++++++++++++++++++++++++---
 src/mon/Monitor.h  |  2 ++
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index aaf61e66014..925d7b02cfb 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -1282,6 +1282,7 @@ void Monitor::bootstrap()
 
   // probe monitors
   dout(10) << "probing other monitors" << dendl;
+  ++probe_epoch;
   for (unsigned i = 0; i < monmap->size(); i++) {
     if ((int)i != rank)
       send_mon_message(
@@ -6335,6 +6336,10 @@ int Monitor::handle_auth_bad_method(
 {
   derr << __func__ << " hmm, they didn't like " << old_auth_method
        << " result " << cpp_strerror(result) << dendl;
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) {
+    /* this is hacky but the least invasive way to cycle secrets: */
+    cycle_mon_secret = probe_epoch;
+  }
   return -EACCES;
 }
 
@@ -6372,9 +6377,18 @@ bool Monitor::get_authorizer(int service_id, AuthAuthorizer **authorizer)
   if (service_id == CEPH_ENTITY_TYPE_MON) {
     // mon to mon authentication uses the private monitor shared key and not the
     // rotating key
-    CryptoKey secret;
-    if (!keyring.get_secret(name, secret) &&
-	!key_server.get_secret(name, secret)) {
+    CryptoKey key_server_secret;
+    CryptoKey keyring_secret;
+
+    bool ksb = key_server.get_secret(name, key_server_secret);
+    if (ksb) {
+      dout(30) << __func__ << ": keyserver found secret=" << key_server_secret << dendl;
+    }
+    bool krb = keyring.get_secret(name, keyring_secret);
+    if (krb) {
+      dout(30) << __func__ << ": keyring found secret=" << keyring_secret << dendl;
+    }
+    if (!ksb && !krb) {
       dout(0) << " couldn't get secret for mon service from keyring or keyserver"
 	      << dendl;
       stringstream ss, ds;
@@ -6387,6 +6401,19 @@ bool Monitor::get_authorizer(int service_id, AuthAuthorizer **authorizer)
       return false;
     }
 
+    CryptoKey secret;
+    dout(30) << __func__ << ": cycle_mon_secret=" << cycle_mon_secret << dendl;
+    if ((((cycle_mon_secret & 1) == 0) && ksb) || !krb) {
+      /* Use KeyServer if present (it should be because Monitor::key_server's
+       * extra_secrets **is** the Monitor::keyring.
+       */
+      dout(15) << __func__ << ": using key_server secret" << dendl;
+      secret = key_server_secret;
+    } else {
+      dout(15) << __func__ << ": using keyring secret" << dendl;
+      secret = keyring_secret;
+    }
+
     ret = key_server.build_session_auth_info(
       service_id, auth_ticket_info.ticket, secret, (uint64_t)-1, secret.get_type(), info);
     if (ret < 0) {
diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index de15e9ae25f..4f54c729efe 100644
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -1132,6 +1132,8 @@ public:
 
 private:
   ceph::coarse_mono_time const starttime = coarse_mono_clock::now();
+  epoch_t probe_epoch = 0;
+  epoch_t cycle_mon_secret = 0;
 };
 
 #define CEPH_MON_FEATURE_INCOMPAT_BASE CompatSet::Feature (1, "initial feature set (~v.18)")
-- 
2.39.5