From 4c283729a769c0c3a7c4fa93ba8d506d239e6ef6 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 28 Sep 2021 13:03:13 -0400 Subject: [PATCH] mon,auth: debug missing service secrets Hunting https://tracker.ceph.com/issues/51815 Signed-off-by: Sage Weil (cherry picked from commit 4bc8ca2e5978a29ef10663a46f7a7050ca7a3a4c) --- src/auth/cephx/CephxKeyServer.cc | 25 +++++++++++++++++++++---- src/auth/cephx/CephxKeyServer.h | 2 ++ src/mon/AuthMonitor.cc | 3 +++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/auth/cephx/CephxKeyServer.cc b/src/auth/cephx/CephxKeyServer.cc index adfe74d2baba6..2961615b13eea 100644 --- a/src/auth/cephx/CephxKeyServer.cc +++ b/src/auth/cephx/CephxKeyServer.cc @@ -72,8 +72,11 @@ bool KeyServerData::get_service_secret(CephContext *cct, uint32_t service_id, uint64_t secret_id, CryptoKey& secret) const { auto iter = rotating_secrets.find(service_id); - if (iter == rotating_secrets.end()) + if (iter == rotating_secrets.end()) { + ldout(cct, 10) << __func__ << " no rotating_secrets for service " << service_id + << " " << ceph_entity_type_name(service_id) << dendl; return false; + } const RotatingSecrets& secrets = iter->second; auto riter = secrets.secrets.find(secret_id); @@ -151,6 +154,11 @@ int KeyServer::start_server() return 0; } +void KeyServer::dump() +{ + _dump_rotating_secrets(); +} + bool KeyServer::_check_rotating_secrets() { ldout(cct, 10) << "_check_rotating_secrets" << dendl; @@ -163,8 +171,10 @@ bool KeyServer::_check_rotating_secrets() added += _rotate_secret(CEPH_ENTITY_TYPE_MGR); if (added) { - ldout(cct, 10) << __func__ << " added " << added << dendl; data.rotating_ver++; + ldout(cct, 10) << __func__ << " added " << added + << ", rotating_ver=" << data.rotating_ver + << dendl; //data.next_rotating_time = ceph_clock_now(cct); //data.next_rotating_time += std::min(cct->_conf->auth_mon_ticket_ttl, cct->_conf->auth_service_ticket_ttl); _dump_rotating_secrets(); @@ -359,11 +369,18 @@ void KeyServer::encode_plaintext(bufferlist &bl) bool KeyServer::updated_rotating(bufferlist& rotating_bl, version_t& rotating_ver) { std::scoped_lock l{lock}; + ldout(cct, 20) << __func__ << " before: data.rotating_ver=" << data.rotating_ver + << " vs rotating_ver " << rotating_ver << dendl; - _check_rotating_secrets(); + bool r = _check_rotating_secrets(); + + ldout(cct, 20) << __func__ << " after: data.rotating_ver=" << data.rotating_ver + << " vs rotating_ver " << rotating_ver << dendl; - if (data.rotating_ver <= rotating_ver) + if (data.rotating_ver <= rotating_ver) { + ceph_assert(!r); return false; + } data.encode_rotating(rotating_bl); diff --git a/src/auth/cephx/CephxKeyServer.h b/src/auth/cephx/CephxKeyServer.h index 3576c42c2357a..68c40cc01b075 100644 --- a/src/auth/cephx/CephxKeyServer.h +++ b/src/auth/cephx/CephxKeyServer.h @@ -215,6 +215,8 @@ public: int start_server(); void rotate_timeout(double timeout); + void dump(); + int build_session_auth_info(uint32_t service_id, const AuthTicket& parent_ticket, CephXSessionAuthInfo& info); diff --git a/src/mon/AuthMonitor.cc b/src/mon/AuthMonitor.cc index 1ffa80a0d053f..a4ae8c82f464d 100644 --- a/src/mon/AuthMonitor.cc +++ b/src/mon/AuthMonitor.cc @@ -358,7 +358,10 @@ void AuthMonitor::update_from_paxos(bool *need_bootstrap) dout(10) << __func__ << " max_global_id=" << max_global_id << " format_version " << format_version + << ", last_rotating_ver " << last_rotating_ver << dendl; + + mon.key_server.dump(); } bool AuthMonitor::_should_increase_max_global_id() -- 2.39.5