mon: cycle through keyring or key_server for auth with mons

After commit `mon: use key_server for looking up mon key`, the mons will now
use the key_server to lookup the `mon.` key when a mon connects.  We need to
make the mons prefer using that key with authenticating during probing other
mons. However, the protocol doesn't allow falling back to another key. This is
necessary if what's in the key_server database is out-of-date due to an earlier
loss of quorum. In that case, the operator should update the local keyring file
and the mon should give that a try if auth fails.

Signed-off-by: Patrick Donnelly <pdonnell@ibm.com>

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 63c02d920dd5a23ac5372106d01b360d59d1b827..6bfdccb12d6769e36eee5013147fb24b38d0721b 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -1286,6 +1286,7 @@ void Monitor::bootstrap()
 
   // probe monitors
   dout(10) << "probing other monitors" << dendl;
+  ++probe_epoch;
   for (unsigned i = 0; i < monmap->size(); i++) {
     if ((int)i != rank)
       send_mon_message(
@@ -6356,6 +6357,10 @@ int Monitor::handle_auth_bad_method(
 {
   derr << __func__ << " hmm, they didn't like " << old_auth_method
        << " result " << cpp_strerror(result) << dendl;
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) {
+    /* this is hacky but the least invasive way to cycle secrets: */
+    cycle_mon_secret = probe_epoch;
+  }
   return -EACCES;
 }
 
@@ -6393,9 +6398,18 @@ bool Monitor::get_authorizer(int service_id, AuthAuthorizer **authorizer)
   if (service_id == CEPH_ENTITY_TYPE_MON) {
     // mon to mon authentication uses the private monitor shared key and not the
     // rotating key
-    CryptoKey secret;
-    if (!keyring.get_secret(name, secret) &&
-       !key_server.get_secret(name, secret)) {
+    CryptoKey key_server_secret;
+    CryptoKey keyring_secret;
+
+    bool ksb = key_server.get_secret(name, key_server_secret);
+    if (ksb) {
+      dout(30) << __func__ << ": keyserver found secret=" << key_server_secret << dendl;
+    }
+    bool krb = keyring.get_secret(name, keyring_secret);
+    if (krb) {
+      dout(30) << __func__ << ": keyring found secret=" << keyring_secret << dendl;
+    }
+    if (!ksb && !krb) {
       dout(0) << " couldn't get secret for mon service from keyring or keyserver"
              << dendl;
       stringstream ss, ds;
@@ -6408,6 +6422,19 @@ bool Monitor::get_authorizer(int service_id, AuthAuthorizer **authorizer)
       return false;
     }
 
+    CryptoKey secret;
+    dout(30) << __func__ << ": cycle_mon_secret=" << cycle_mon_secret << dendl;
+    if ((((cycle_mon_secret & 1) == 0) && ksb) || !krb) {
+      /* Use KeyServer if present (it should be because Monitor::key_server's
+       * extra_secrets **is** the Monitor::keyring.
+       */
+      dout(15) << __func__ << ": using key_server secret" << dendl;
+      secret = key_server_secret;
+    } else {
+      dout(15) << __func__ << ": using keyring secret" << dendl;
+      secret = keyring_secret;
+    }
+
     ret = key_server.build_session_auth_info(
       service_id, auth_ticket_info.ticket, secret, (uint64_t)-1, secret.get_type(), info);
     if (ret < 0) {

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 2d22cb144c681b9baf40d6400314355be6311f7f..f4ded8fa17c6f7f91ac54fc2faf1671f7f393129 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -1133,6 +1133,8 @@ public:
 
 private:
   ceph::coarse_mono_time const starttime = coarse_mono_clock::now();
+  epoch_t probe_epoch = 0;
+  epoch_t cycle_mon_secret = 0;
 };
 
 #define CEPH_MON_FEATURE_INCOMPAT_BASE CompatSet::Feature (1, "initial feature set (~v.18)")