mon,auth: debug missing service secrets

Hunting https://tracker.ceph.com/issues/51815

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/src/auth/cephx/CephxKeyServer.cc b/src/auth/cephx/CephxKeyServer.cc
index 89e0c415c8a757452086081e8480b431d4e52d68..17c6e091887835f228b19b3728bd10f73efc9642 100644 (file)
--- a/src/auth/cephx/CephxKeyServer.cc
+++ b/src/auth/cephx/CephxKeyServer.cc
@@ -72,8 +72,11 @@ bool KeyServerData::get_service_secret(CephContext *cct, uint32_t service_id,
                                uint64_t secret_id, CryptoKey& secret) const
 {
   auto iter = rotating_secrets.find(service_id);
-  if (iter == rotating_secrets.end())
+  if (iter == rotating_secrets.end()) {
+    ldout(cct, 10) << __func__ << " no rotating_secrets for service " << service_id
+                  << " " << ceph_entity_type_name(service_id) << dendl;
     return false;
+  }
 
   const RotatingSecrets& secrets = iter->second;
   auto riter = secrets.secrets.find(secret_id);
@@ -151,6 +154,11 @@ int KeyServer::start_server()
   return 0;
 }
 
+void KeyServer::dump()
+{
+  _dump_rotating_secrets();
+}
+
 bool KeyServer::_check_rotating_secrets()
 {
   ldout(cct, 10) << "_check_rotating_secrets" << dendl;
@@ -163,8 +171,10 @@ bool KeyServer::_check_rotating_secrets()
   added += _rotate_secret(CEPH_ENTITY_TYPE_MGR);
 
   if (added) {
-    ldout(cct, 10) << __func__ << " added " << added << dendl;
     data.rotating_ver++;
+    ldout(cct, 10) << __func__ << " added " << added
+                  << ", rotating_ver=" << data.rotating_ver
+                  << dendl;
     //data.next_rotating_time = ceph_clock_now(cct);
     //data.next_rotating_time += std::min(cct->_conf->auth_mon_ticket_ttl, cct->_conf->auth_service_ticket_ttl);
     _dump_rotating_secrets();
@@ -359,11 +369,18 @@ void KeyServer::encode_plaintext(bufferlist &bl)
 bool KeyServer::updated_rotating(bufferlist& rotating_bl, version_t& rotating_ver)
 {
   std::scoped_lock l{lock};
+  ldout(cct, 20) << __func__ << " before: data.rotating_ver=" << data.rotating_ver
+                << " vs rotating_ver " << rotating_ver << dendl;
 
-  _check_rotating_secrets(); 
+  bool r = _check_rotating_secrets();
+  
+  ldout(cct, 20) << __func__ << " after: data.rotating_ver=" << data.rotating_ver
+                << " vs rotating_ver " << rotating_ver << dendl;
 
-  if (data.rotating_ver <= rotating_ver)
+  if (data.rotating_ver <= rotating_ver) {
+    ceph_assert(!r);
     return false;
+  }
  
   data.encode_rotating(rotating_bl);
 

diff --git a/src/auth/cephx/CephxKeyServer.h b/src/auth/cephx/CephxKeyServer.h
index 3576c42c2357adad37bc1ce804d45558240c51ba..68c40cc01b075a165e1bcfd2a60d6d9a7510488b 100644 (file)
--- a/src/auth/cephx/CephxKeyServer.h
+++ b/src/auth/cephx/CephxKeyServer.h
@@ -215,6 +215,8 @@ public:
   int start_server();
   void rotate_timeout(double timeout);
 
+  void dump();
+  
   int build_session_auth_info(uint32_t service_id,
                              const AuthTicket& parent_ticket,
                              CephXSessionAuthInfo& info);

diff --git a/src/mon/AuthMonitor.cc b/src/mon/AuthMonitor.cc
index bf5b18d5ff9cd3643bc920b8b128c77f23d60946..ebe99c6724264a25973a859659265943636848ea 100644 (file)
--- a/src/mon/AuthMonitor.cc
+++ b/src/mon/AuthMonitor.cc
@@ -358,7 +358,10 @@ void AuthMonitor::update_from_paxos(bool *need_bootstrap)
 
   dout(10) << __func__ << " max_global_id=" << max_global_id
           << " format_version " << format_version
+          << ", last_rotating_ver " << last_rotating_ver
           << dendl;
+
+  mon.key_server.dump();
 }
 
 bool AuthMonitor::_should_increase_max_global_id()