monclient: avoid key renew storm on clock skew

Refreshing rotating keys too often is a symptom of a clock skew, try to
detect it and don't cause extra problems:

* MonClient::_check_auth_rotating:
  - detect and report premature keys expiration due to a time skew
  - rate limit refreshing the keys to avoid excessive RAM and CPU usage
    (both by OSD in question and monitors which have to process a lot
    of auth messages)
* MonClient::wait_auth_rotating: wait for valid (not expired) keys
* OSD::init(): bail out after 10 attempts to obtain the rotating keys

Fixes: #12065
Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>

diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index dc1ec6c4a7b0d99aeace3551a5062c28b926c2a6..87011fbdb8bd05b09aa0eace1b01b7aea3e67a91 100644 (file)
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -534,6 +534,7 @@ void MonClient::handle_auth(MAuthReply *m)
   if (ret == 0) {
     if (state != MC_STATE_HAVE_SESSION) {
       state = MC_STATE_HAVE_SESSION;
+      last_rotating_renew_sent = utime_t();
       while (!waiting_for_session.empty()) {
        _send_mon_message(waiting_for_session.front());
        waiting_for_session.pop_front();
@@ -831,8 +832,11 @@ int MonClient::_check_auth_rotating()
     return 0;
   }
 
-  utime_t cutoff = ceph_clock_now(cct);
+  utime_t now = ceph_clock_now(cct);
+  utime_t cutoff = now;
   cutoff -= MIN(30.0, cct->_conf->auth_service_ticket_ttl / 4.0);
+  utime_t issued_at_lower_bound = now;
+  issued_at_lower_bound -= cct->_conf->auth_service_ticket_ttl;
   if (!rotating_secrets->need_new_secrets(cutoff)) {
     ldout(cct, 10) << "_check_auth_rotating have uptodate secrets (they expire after " << cutoff << ")" << dendl;
     rotating_secrets->dump_rotating();
@@ -840,9 +844,22 @@ int MonClient::_check_auth_rotating()
   }
 
   ldout(cct, 10) << "_check_auth_rotating renewing rotating keys (they expired before " << cutoff << ")" << dendl;
+  if (!rotating_secrets->need_new_secrets() &&
+      rotating_secrets->need_new_secrets(issued_at_lower_bound)) {
+    // the key has expired before it has been issued?
+    lderr(cct) << __func__ << " possible clock skew, rotating keys expired way too early"
+               << " (before " << issued_at_lower_bound << ")" << dendl;
+  }
+  if ((now > last_rotating_renew_sent) &&
+      double(now - last_rotating_renew_sent) < 1) {
+    ldout(cct, 10) << __func__ << " called too often (last: "
+                   << last_rotating_renew_sent << "), skipping refresh" << dendl;
+    return 0;
+  }
   MAuth *m = new MAuth;
   m->protocol = auth->get_protocol();
   if (auth->build_rotating_request(m->auth_payload)) {
+    last_rotating_renew_sent = now;
     _send_mon_message(m);
   } else {
     m->put();
@@ -853,7 +870,8 @@ int MonClient::_check_auth_rotating()
 int MonClient::wait_auth_rotating(double timeout)
 {
   Mutex::Locker l(monc_lock);
-  utime_t until = ceph_clock_now(cct);
+  utime_t now = ceph_clock_now(cct);
+  utime_t until = now;
   until += timeout;
 
   if (auth->get_protocol() == CEPH_AUTH_NONE)
@@ -863,14 +881,14 @@ int MonClient::wait_auth_rotating(double timeout)
     return 0;
 
   while (auth_principal_needs_rotating_keys(entity_name) &&
-        rotating_secrets->need_new_secrets()) {
-    utime_t now = ceph_clock_now(cct);
+        rotating_secrets->need_new_secrets(now)) {
     if (now >= until) {
       ldout(cct, 0) << "wait_auth_rotating timed out after " << timeout << dendl;
       return -ETIMEDOUT;
     }
     ldout(cct, 10) << "wait_auth_rotating waiting (until " << until << ")" << dendl;
     auth_cond.WaitUntil(monc_lock, until);
+    now = ceph_clock_now(cct);
   }
   ldout(cct, 10) << "wait_auth_rotating done" << dendl;
   return 0;

diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h
index d98a15b61aaba7fbe8ea6624a9df6b3cebedae72..2efa1ff48facc2d9171da5113032e1693c78f67d 100644 (file)
--- a/src/mon/MonClient.h
+++ b/src/mon/MonClient.h
@@ -179,6 +179,7 @@ private:
   int authenticate_err;
 
   list<Message*> waiting_for_session;
+  utime_t last_rotating_renew_sent;
   Context *session_established_context;
   bool had_a_connection;
   double reopen_interval_multiplier;

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 57242a48b46f8476fa5d539a8c2945e0fc74105a..0036290eb9349cd89704e879ceffd23e6b44a19d 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2028,6 +2028,9 @@ int OSD::init()
     daily_loadavg = 1.0;
   }
 
+  int rotating_auth_attempts = 0;
+  const int max_rotating_auth_attempts = 10;
+
   // read superblock
   r = read_superblock();
   if (r < 0) {
@@ -2177,6 +2180,14 @@ int OSD::init()
 
   while (monc->wait_auth_rotating(30.0) < 0) {
     derr << "unable to obtain rotating service keys; retrying" << dendl;
+    ++rotating_auth_attempts;
+    if (rotating_auth_attempts > max_rotating_auth_attempts) {
+        osd_lock.Lock(); // make locker happy
+        if (!is_stopping()) {
+            r = - ETIMEDOUT;
+        }
+        goto monout;
+    }
   }
 
   osd_lock.Lock();