From ee86927f84a9372eb433b6543ef3b93fbd01f597 Mon Sep 17 00:00:00 2001 From: Prashant D Date: Mon, 28 Mar 2022 14:02:08 +0100 Subject: [PATCH] mgr, mon: Keep upto date metadata with mgr for MONs The mgr updates mon metadata through handle_mon_map which gets triggered when MONs were removed/added from/to cluster or the active mgr is restarted or mgr failsover. We could have handled metadata update through MMgrOpen or early MMgrReport messages but these are sent before monitor electin completes and lead monitor updates pending metadata in monstore. Instead of relying on fetching mon metadata using 'ceph mon metadata ' command, explicitly send metadata update request with mon metadata to mgr. Fixes: https://tracker.ceph.com/issues/55088 Signed-off-by: Prashant D (cherry picked from commit 1a065043b964f8c014ebb5bc890a243c398ff07c) --- src/messages/MMgrUpdate.h | 92 +++++++++++++++++++++++++++++++++++++++ src/mgr/DaemonServer.cc | 46 ++++++++++++++++++++ src/mgr/DaemonServer.h | 2 + src/mgr/MgrClient.cc | 45 +++++++++++++++++++ src/mgr/MgrClient.h | 6 +++ src/mon/Monitor.cc | 13 ++++++ src/mon/Monitor.h | 2 + src/mon/MonmapMonitor.cc | 2 + src/msg/Message.cc | 5 +++ src/msg/Message.h | 3 ++ src/msg/MessageRef.h | 1 + 11 files changed, 217 insertions(+) create mode 100644 src/messages/MMgrUpdate.h diff --git a/src/messages/MMgrUpdate.h b/src/messages/MMgrUpdate.h new file mode 100644 index 0000000000000..71a528d3150bf --- /dev/null +++ b/src/messages/MMgrUpdate.h @@ -0,0 +1,92 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2022 Prashant D + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + + +#ifndef CEPH_MMGRUPDATE_H_ +#define CEPH_MMGRUPDATE_H_ + +#include "msg/Message.h" + +class MMgrUpdate : public Message { +private: + static constexpr int HEAD_VERSION = 3; + static constexpr int COMPAT_VERSION = 1; + +public: + + std::string daemon_name; + std::string service_name; // optional; otherwise infer from entity type + + bool service_daemon = false; + std::map daemon_metadata; + std::map daemon_status; + + bool need_metadata_update = false; + + void decode_payload() override + { + using ceph::decode; + auto p = payload.cbegin(); + decode(daemon_name, p); + if (header.version >= 2) { + decode(service_name, p); + decode(service_daemon, p); + if (service_daemon) { + decode(daemon_metadata, p); + decode(daemon_status, p); + } + } + if (header.version >= 3) { + decode(need_metadata_update, p); + } + } + + void encode_payload(uint64_t features) override { + using ceph::encode; + encode(daemon_name, payload); + encode(service_name, payload); + encode(service_daemon, payload); + if (service_daemon) { + encode(daemon_metadata, payload); + encode(daemon_status, payload); + } + encode(need_metadata_update, payload); + } + + std::string_view get_type_name() const override { return "mgrupdate"; } + void print(std::ostream& out) const override { + out << get_type_name() << "("; + if (service_name.length()) { + out << service_name; + } else { + out << ceph_entity_type_name(get_source().type()); + } + out << "." << daemon_name; + if (service_daemon) { + out << " daemon"; + } + out << ")"; + } + +private: + MMgrUpdate() + : Message{MSG_MGR_UPDATE, HEAD_VERSION, COMPAT_VERSION} + {} + using RefCountedObject::put; + using RefCountedObject::get; + template + friend boost::intrusive_ptr ceph::make_message(Args&&... args); +}; + +#endif + diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc index b1a3c5c8e4a5d..1ab00e0afd6f3 100644 --- a/src/mgr/DaemonServer.cc +++ b/src/mgr/DaemonServer.cc @@ -27,6 +27,7 @@ #include "mon/MonCommand.h" #include "messages/MMgrOpen.h" +#include "messages/MMgrUpdate.h" #include "messages/MMgrClose.h" #include "messages/MMgrConfigure.h" #include "messages/MMonMgrReport.h" @@ -258,6 +259,8 @@ bool DaemonServer::ms_dispatch2(const ref_t& m) return handle_report(ref_cast(m)); case MSG_MGR_OPEN: return handle_open(ref_cast(m)); + case MSG_MGR_UPDATE: + return handle_update(ref_cast(m)); case MSG_MGR_CLOSE: return handle_close(ref_cast(m)); case MSG_COMMAND: @@ -522,6 +525,49 @@ bool DaemonServer::handle_open(const ref_t& m) return true; } +bool DaemonServer::handle_update(const ref_t& m) +{ + DaemonKey key; + if (!m->service_name.empty()) { + key.type = m->service_name; + } else { + key.type = ceph_entity_type_name(m->get_connection()->get_peer_type()); + } + key.name = m->daemon_name; + + dout(10) << "from " << m->get_connection() << " " << key << dendl; + + if (m->get_connection()->get_peer_type() == entity_name_t::TYPE_CLIENT && + m->service_name.empty()) { + // Clients should not be sending us update request + dout(10) << "rejecting update request from non-daemon client " << m->daemon_name + << dendl; + clog->warn() << "rejecting report from non-daemon client " << m->daemon_name + << " at " << m->get_connection()->get_peer_addrs(); + m->get_connection()->mark_down(); + return true; + } + + + { + std::unique_lock locker(lock); + + DaemonStatePtr daemon; + // Look up the DaemonState + if (daemon_state.exists(key)) { + dout(20) << "updating existing DaemonState for " << key << dendl; + + daemon = daemon_state.get(key); + if (m->need_metadata_update == true && + !m->daemon_metadata.empty()) { + daemon_state.update_metadata(daemon, m->daemon_metadata); + } + } + } + + return true; +} + bool DaemonServer::handle_close(const ref_t& m) { std::lock_guard l(lock); diff --git a/src/mgr/DaemonServer.h b/src/mgr/DaemonServer.h index 3adcf6ccaf93b..c746498a56b6f 100644 --- a/src/mgr/DaemonServer.h +++ b/src/mgr/DaemonServer.h @@ -36,6 +36,7 @@ class MMgrReport; class MMgrOpen; +class MMgrUpdate; class MMgrClose; class MMonMgrReport; class MCommand; @@ -275,6 +276,7 @@ public: void fetch_missing_metadata(const DaemonKey& key, const entity_addr_t& addr); bool handle_open(const ceph::ref_t& m); + bool handle_update(const ceph::ref_t& m); bool handle_close(const ceph::ref_t& m); bool handle_report(const ceph::ref_t& m); bool handle_command(const ceph::ref_t& m); diff --git a/src/mgr/MgrClient.cc b/src/mgr/MgrClient.cc index 4ac88dd99d462..626c9941a09da 100644 --- a/src/mgr/MgrClient.cc +++ b/src/mgr/MgrClient.cc @@ -21,6 +21,7 @@ #include "messages/MMgrMap.h" #include "messages/MMgrReport.h" #include "messages/MMgrOpen.h" +#include "messages/MMgrUpdate.h" #include "messages/MMgrClose.h" #include "messages/MMgrConfigure.h" #include "messages/MCommand.h" @@ -245,6 +246,25 @@ void MgrClient::_send_open() } } +void MgrClient::_send_update() +{ + if (session && session->con) { + auto update = make_message(); + if (!service_name.empty()) { + update->service_name = service_name; + update->daemon_name = daemon_name; + } else { + update->daemon_name = cct->_conf->name.get_id(); + } + if (service_daemon) { + update->service_daemon = service_daemon; + update->daemon_metadata = daemon_metadata; + } + update->need_metadata_update = need_metadata_update; + session->con->send_message2(update); + } +} + bool MgrClient::handle_mgr_map(ref_t m) { ceph_assert(ceph_mutex_is_locked_by_me(lock)); @@ -568,6 +588,31 @@ bool MgrClient::handle_command_reply( return true; } +int MgrClient::update_daemon_metadata( + const std::string& service, + const std::string& name, + const std::map& metadata) +{ + std::lock_guard l(lock); + if (service_daemon) { + return -EEXIST; + } + ldout(cct,1) << service << "." << name << " metadata " << metadata << dendl; + service_daemon = true; + service_name = service; + daemon_name = name; + daemon_metadata = metadata; + daemon_dirty_status = true; + + if (need_metadata_update == true && + !daemon_metadata.empty()) { + _send_update(); + need_metadata_update = false; + } + + return 0; +} + int MgrClient::service_daemon_register( const std::string& service, const std::string& name, diff --git a/src/mgr/MgrClient.h b/src/mgr/MgrClient.h index 6bd2b26e11a37..1668d8da0fbc2 100644 --- a/src/mgr/MgrClient.h +++ b/src/mgr/MgrClient.h @@ -94,6 +94,7 @@ protected: bool service_daemon = false; bool daemon_dirty_status = false; bool task_dirty_status = false; + bool need_metadata_update = true; std::string service_name, daemon_name; std::map daemon_metadata; std::map daemon_status; @@ -102,6 +103,7 @@ protected: void reconnect(); void _send_open(); + void _send_update(); // In pre-luminous clusters, the ceph-mgr service is absent or optional, // so we must not block in start_command waiting for it. @@ -157,6 +159,10 @@ public: ceph::buffer::list *outbl, std::string *outs, Context *onfinish); + int update_daemon_metadata( + const std::string& service, + const std::string& name, + const std::map& metadata); int service_daemon_register( const std::string& service, const std::string& name, diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index ab9d49acce224..aee9198a936d6 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -2966,6 +2966,19 @@ void Monitor::log_health( } } +void Monitor::update_pending_metadata() +{ + Metadata metadata; + collect_metadata(&metadata); + size_t version_size = mon_metadata[rank]["ceph_version_short"].size(); + const std::string current_version = mon_metadata[rank]["ceph_version_short"]; + const std::string pending_version = metadata["ceph_version_short"]; + + if (current_version.compare(0, version_size, pending_version) < 0) { + mgr_client.update_daemon_metadata("mon", name, metadata); + } +} + void Monitor::get_cluster_status(stringstream &ss, Formatter *f, MonSession *session) { diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index e16dc4a323a32..1f7e64e8592cb 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -796,6 +796,8 @@ public: const health_check_map_t& previous, MonitorDBStore::TransactionRef t); + void update_pending_metadata(); + protected: class HealthCheckLogStatus { diff --git a/src/mon/MonmapMonitor.cc b/src/mon/MonmapMonitor.cc index c9dfa3af499b1..1ecbea8578c27 100644 --- a/src/mon/MonmapMonitor.cc +++ b/src/mon/MonmapMonitor.cc @@ -248,6 +248,8 @@ void MonmapMonitor::on_active() apply_mon_features(mon.get_quorum_mon_features(), mon.quorum_min_mon_release); + + mon.update_pending_metadata(); } bool MonmapMonitor::preprocess_query(MonOpRequestRef op) diff --git a/src/msg/Message.cc b/src/msg/Message.cc index 6c57d355bdf49..266eb7676786b 100644 --- a/src/msg/Message.cc +++ b/src/msg/Message.cc @@ -190,6 +190,7 @@ #include "messages/MMgrDigest.h" #include "messages/MMgrReport.h" #include "messages/MMgrOpen.h" +#include "messages/MMgrUpdate.h" #include "messages/MMgrClose.h" #include "messages/MMgrConfigure.h" #include "messages/MMonMgrReport.h" @@ -893,6 +894,10 @@ Message *decode_message(CephContext *cct, m = make_message(); break; + case MSG_MGR_UPDATE: + m = make_message(); + break; + case MSG_MGR_CLOSE: m = make_message(); break; diff --git a/src/msg/Message.h b/src/msg/Message.h index 8e1c4f967f059..c3b47eee20b07 100644 --- a/src/msg/Message.h +++ b/src/msg/Message.h @@ -235,6 +235,9 @@ #define MSG_MGR_COMMAND 0x709 #define MSG_MGR_COMMAND_REPLY 0x70a +// *** ceph-mgr <-> MON daemons *** +#define MSG_MGR_UPDATE 0x70b + // ====================================================== // abstract Message class diff --git a/src/msg/MessageRef.h b/src/msg/MessageRef.h index 2c11aced52b34..6bed38311c148 100644 --- a/src/msg/MessageRef.h +++ b/src/msg/MessageRef.h @@ -97,6 +97,7 @@ class MMgrConfigure; class MMgrDigest; class MMgrMap; class MMgrOpen; +class MMgrUpdate; class MMgrReport; class MMonCommandAck; class MMonCommand; -- 2.39.5