From 0da036fb1908fb15676c02db791535df87d6e2ea Mon Sep 17 00:00:00 2001 From: Prashant D Date: Mon, 28 Mar 2022 14:02:08 +0100 Subject: [PATCH] mgr, mon: Keep upto date metadata with mgr for MONs The mgr updates mon metadata through handle_mon_map which gets triggered when MONs were removed/added from/to cluster or the active mgr is restarted or mgr failsover. We could have handled metadata update through MMgrOpen or early MMgrReport messages but these are sent before monitor electin completes and lead monitor updates pending metadata in monstore. Instead of relying on fetching mon metadata using 'ceph mon metadata ' command, explicitly send metadata update request with mon metadata to mgr. Fixes: https://tracker.ceph.com/issues/55088 Signed-off-by: Prashant D (cherry picked from commit 1a065043b964f8c014ebb5bc890a243c398ff07c) --- src/messages/MMgrUpdate.h | 92 +++++++++++++++++++++++++++++++++++++++ src/mgr/DaemonServer.cc | 46 ++++++++++++++++++++ src/mgr/DaemonServer.h | 2 + src/mgr/MgrClient.cc | 45 +++++++++++++++++++ src/mgr/MgrClient.h | 6 +++ src/mon/Monitor.cc | 13 ++++++ src/mon/Monitor.h | 2 + src/mon/MonmapMonitor.cc | 2 + src/msg/Message.cc | 5 +++ src/msg/Message.h | 3 ++ src/msg/MessageRef.h | 1 + 11 files changed, 217 insertions(+) create mode 100644 src/messages/MMgrUpdate.h diff --git a/src/messages/MMgrUpdate.h b/src/messages/MMgrUpdate.h new file mode 100644 index 000000000000..71a528d3150b --- /dev/null +++ b/src/messages/MMgrUpdate.h @@ -0,0 +1,92 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2022 Prashant D + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + + +#ifndef CEPH_MMGRUPDATE_H_ +#define CEPH_MMGRUPDATE_H_ + +#include "msg/Message.h" + +class MMgrUpdate : public Message { +private: + static constexpr int HEAD_VERSION = 3; + static constexpr int COMPAT_VERSION = 1; + +public: + + std::string daemon_name; + std::string service_name; // optional; otherwise infer from entity type + + bool service_daemon = false; + std::map daemon_metadata; + std::map daemon_status; + + bool need_metadata_update = false; + + void decode_payload() override + { + using ceph::decode; + auto p = payload.cbegin(); + decode(daemon_name, p); + if (header.version >= 2) { + decode(service_name, p); + decode(service_daemon, p); + if (service_daemon) { + decode(daemon_metadata, p); + decode(daemon_status, p); + } + } + if (header.version >= 3) { + decode(need_metadata_update, p); + } + } + + void encode_payload(uint64_t features) override { + using ceph::encode; + encode(daemon_name, payload); + encode(service_name, payload); + encode(service_daemon, payload); + if (service_daemon) { + encode(daemon_metadata, payload); + encode(daemon_status, payload); + } + encode(need_metadata_update, payload); + } + + std::string_view get_type_name() const override { return "mgrupdate"; } + void print(std::ostream& out) const override { + out << get_type_name() << "("; + if (service_name.length()) { + out << service_name; + } else { + out << ceph_entity_type_name(get_source().type()); + } + out << "." << daemon_name; + if (service_daemon) { + out << " daemon"; + } + out << ")"; + } + +private: + MMgrUpdate() + : Message{MSG_MGR_UPDATE, HEAD_VERSION, COMPAT_VERSION} + {} + using RefCountedObject::put; + using RefCountedObject::get; + template + friend boost::intrusive_ptr ceph::make_message(Args&&... args); +}; + +#endif + diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc index c44c952c7c19..b8120a14a7ec 100644 --- a/src/mgr/DaemonServer.cc +++ b/src/mgr/DaemonServer.cc @@ -27,6 +27,7 @@ #include "mon/MonCommand.h" #include "messages/MMgrOpen.h" +#include "messages/MMgrUpdate.h" #include "messages/MMgrClose.h" #include "messages/MMgrConfigure.h" #include "messages/MMonMgrReport.h" @@ -267,6 +268,8 @@ bool DaemonServer::ms_dispatch2(const ref_t& m) return handle_report(ref_cast(m)); case MSG_MGR_OPEN: return handle_open(ref_cast(m)); + case MSG_MGR_UPDATE: + return handle_update(ref_cast(m)); case MSG_MGR_CLOSE: return handle_close(ref_cast(m)); case MSG_COMMAND: @@ -531,6 +534,49 @@ bool DaemonServer::handle_open(const ref_t& m) return true; } +bool DaemonServer::handle_update(const ref_t& m) +{ + DaemonKey key; + if (!m->service_name.empty()) { + key.type = m->service_name; + } else { + key.type = ceph_entity_type_name(m->get_connection()->get_peer_type()); + } + key.name = m->daemon_name; + + dout(10) << "from " << m->get_connection() << " " << key << dendl; + + if (m->get_connection()->get_peer_type() == entity_name_t::TYPE_CLIENT && + m->service_name.empty()) { + // Clients should not be sending us update request + dout(10) << "rejecting update request from non-daemon client " << m->daemon_name + << dendl; + clog->warn() << "rejecting report from non-daemon client " << m->daemon_name + << " at " << m->get_connection()->get_peer_addrs(); + m->get_connection()->mark_down(); + return true; + } + + + { + std::unique_lock locker(lock); + + DaemonStatePtr daemon; + // Look up the DaemonState + if (daemon_state.exists(key)) { + dout(20) << "updating existing DaemonState for " << key << dendl; + + daemon = daemon_state.get(key); + if (m->need_metadata_update == true && + !m->daemon_metadata.empty()) { + daemon_state.update_metadata(daemon, m->daemon_metadata); + } + } + } + + return true; +} + bool DaemonServer::handle_close(const ref_t& m) { std::lock_guard l(lock); diff --git a/src/mgr/DaemonServer.h b/src/mgr/DaemonServer.h index d719a96a7bcc..ff98356807ad 100644 --- a/src/mgr/DaemonServer.h +++ b/src/mgr/DaemonServer.h @@ -36,6 +36,7 @@ class MMgrReport; class MMgrOpen; +class MMgrUpdate; class MMgrClose; class MMonMgrReport; class MCommand; @@ -275,6 +276,7 @@ public: void fetch_missing_metadata(const DaemonKey& key, const entity_addr_t& addr); bool handle_open(const ceph::ref_t& m); + bool handle_update(const ceph::ref_t& m); bool handle_close(const ceph::ref_t& m); bool handle_report(const ceph::ref_t& m); bool handle_command(const ceph::ref_t& m); diff --git a/src/mgr/MgrClient.cc b/src/mgr/MgrClient.cc index 66ac5901e509..cfb2e99672ec 100644 --- a/src/mgr/MgrClient.cc +++ b/src/mgr/MgrClient.cc @@ -21,6 +21,7 @@ #include "messages/MMgrMap.h" #include "messages/MMgrReport.h" #include "messages/MMgrOpen.h" +#include "messages/MMgrUpdate.h" #include "messages/MMgrClose.h" #include "messages/MMgrConfigure.h" #include "messages/MCommand.h" @@ -245,6 +246,25 @@ void MgrClient::_send_open() } } +void MgrClient::_send_update() +{ + if (session && session->con) { + auto update = make_message(); + if (!service_name.empty()) { + update->service_name = service_name; + update->daemon_name = daemon_name; + } else { + update->daemon_name = cct->_conf->name.get_id(); + } + if (service_daemon) { + update->service_daemon = service_daemon; + update->daemon_metadata = daemon_metadata; + } + update->need_metadata_update = need_metadata_update; + session->con->send_message2(update); + } +} + bool MgrClient::handle_mgr_map(ref_t m) { ceph_assert(ceph_mutex_is_locked_by_me(lock)); @@ -566,6 +586,31 @@ bool MgrClient::handle_command_reply( return true; } +int MgrClient::update_daemon_metadata( + const std::string& service, + const std::string& name, + const std::map& metadata) +{ + std::lock_guard l(lock); + if (service_daemon) { + return -EEXIST; + } + ldout(cct,1) << service << "." << name << " metadata " << metadata << dendl; + service_daemon = true; + service_name = service; + daemon_name = name; + daemon_metadata = metadata; + daemon_dirty_status = true; + + if (need_metadata_update == true && + !daemon_metadata.empty()) { + _send_update(); + need_metadata_update = false; + } + + return 0; +} + int MgrClient::service_daemon_register( const std::string& service, const std::string& name, diff --git a/src/mgr/MgrClient.h b/src/mgr/MgrClient.h index f09ba91b712c..a48ae163e18a 100644 --- a/src/mgr/MgrClient.h +++ b/src/mgr/MgrClient.h @@ -94,6 +94,7 @@ protected: bool service_daemon = false; bool daemon_dirty_status = false; bool task_dirty_status = false; + bool need_metadata_update = true; std::string service_name, daemon_name; std::map daemon_metadata; std::map daemon_status; @@ -102,6 +103,7 @@ protected: void reconnect(); void _send_open(); + void _send_update(); // In pre-luminous clusters, the ceph-mgr service is absent or optional, // so we must not block in start_command waiting for it. @@ -157,6 +159,10 @@ public: ceph::buffer::list *outbl, std::string *outs, Context *onfinish); + int update_daemon_metadata( + const std::string& service, + const std::string& name, + const std::map& metadata); int service_daemon_register( const std::string& service, const std::string& name, diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index ac6d6632c39a..b9fc428d3cb3 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -2966,6 +2966,19 @@ void Monitor::log_health( } } +void Monitor::update_pending_metadata() +{ + Metadata metadata; + collect_metadata(&metadata); + size_t version_size = mon_metadata[rank]["ceph_version_short"].size(); + const std::string current_version = mon_metadata[rank]["ceph_version_short"]; + const std::string pending_version = metadata["ceph_version_short"]; + + if (current_version.compare(0, version_size, pending_version) < 0) { + mgr_client.update_daemon_metadata("mon", name, metadata); + } +} + void Monitor::get_cluster_status(stringstream &ss, Formatter *f, MonSession *session) { diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index 454daf63c6c7..575fef0a9c88 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -796,6 +796,8 @@ public: const health_check_map_t& previous, MonitorDBStore::TransactionRef t); + void update_pending_metadata(); + protected: class HealthCheckLogStatus { diff --git a/src/mon/MonmapMonitor.cc b/src/mon/MonmapMonitor.cc index f9792ed57559..6c5a9ce5736a 100644 --- a/src/mon/MonmapMonitor.cc +++ b/src/mon/MonmapMonitor.cc @@ -248,6 +248,8 @@ void MonmapMonitor::on_active() apply_mon_features(mon.get_quorum_mon_features(), mon.quorum_min_mon_release); + + mon.update_pending_metadata(); } bool MonmapMonitor::preprocess_query(MonOpRequestRef op) diff --git a/src/msg/Message.cc b/src/msg/Message.cc index 6c57d355bdf4..266eb7676786 100644 --- a/src/msg/Message.cc +++ b/src/msg/Message.cc @@ -190,6 +190,7 @@ #include "messages/MMgrDigest.h" #include "messages/MMgrReport.h" #include "messages/MMgrOpen.h" +#include "messages/MMgrUpdate.h" #include "messages/MMgrClose.h" #include "messages/MMgrConfigure.h" #include "messages/MMonMgrReport.h" @@ -893,6 +894,10 @@ Message *decode_message(CephContext *cct, m = make_message(); break; + case MSG_MGR_UPDATE: + m = make_message(); + break; + case MSG_MGR_CLOSE: m = make_message(); break; diff --git a/src/msg/Message.h b/src/msg/Message.h index 362ae5ec6089..a7aff1e27a60 100644 --- a/src/msg/Message.h +++ b/src/msg/Message.h @@ -235,6 +235,9 @@ #define MSG_MGR_COMMAND 0x709 #define MSG_MGR_COMMAND_REPLY 0x70a +// *** ceph-mgr <-> MON daemons *** +#define MSG_MGR_UPDATE 0x70b + // ====================================================== // abstract Message class diff --git a/src/msg/MessageRef.h b/src/msg/MessageRef.h index cd5b2dd741a2..ef7a56a44d94 100644 --- a/src/msg/MessageRef.h +++ b/src/msg/MessageRef.h @@ -101,6 +101,7 @@ class MMgrConfigure; class MMgrDigest; class MMgrMap; class MMgrOpen; +class MMgrUpdate; class MMgrReport; class MMonCommandAck; class MMonCommand; -- 2.47.3