From 6122715a09319367270a08d3fafa7f1c814207b6 Mon Sep 17 00:00:00 2001 From: Prashant D Date: Mon, 28 Mar 2022 14:02:08 +0100 Subject: [PATCH] mgr, mon: Keep upto date metadata with mgr for MONs The mgr updates mon metadata through handle_mon_map which gets triggered when MONs were removed/added from/to cluster or the active mgr is restarted or mgr failsover. We could have handled metadata update through MMgrOpen or early MMgrReport messages but these are sent before monitor electin completes and lead monitor updates pending metadata in monstore. Instead of relying on fetching mon metadata using 'ceph mon metadata ' command, explicitly send metadata update request with mon metadata to mgr. Fixes: https://tracker.ceph.com/issues/55088 Resolves: rhbz#2121265 Signed-off-by: Prashant D (cherry picked from commit 1a065043b964f8c014ebb5bc890a243c398ff07c) --- src/messages/MMgrUpdate.h | 92 +++++++++++++++++++++++++++++++++++++++ src/mgr/DaemonServer.cc | 46 ++++++++++++++++++++ src/mgr/DaemonServer.h | 2 + src/mgr/MgrClient.cc | 45 +++++++++++++++++++ src/mgr/MgrClient.h | 6 +++ src/mon/Monitor.cc | 13 ++++++ src/mon/Monitor.h | 2 + src/mon/MonmapMonitor.cc | 2 + src/msg/Message.cc | 5 +++ src/msg/Message.h | 3 ++ src/msg/MessageRef.h | 1 + 11 files changed, 217 insertions(+) create mode 100644 src/messages/MMgrUpdate.h diff --git a/src/messages/MMgrUpdate.h b/src/messages/MMgrUpdate.h new file mode 100644 index 0000000000000..71a528d3150bf --- /dev/null +++ b/src/messages/MMgrUpdate.h @@ -0,0 +1,92 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2022 Prashant D + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + + +#ifndef CEPH_MMGRUPDATE_H_ +#define CEPH_MMGRUPDATE_H_ + +#include "msg/Message.h" + +class MMgrUpdate : public Message { +private: + static constexpr int HEAD_VERSION = 3; + static constexpr int COMPAT_VERSION = 1; + +public: + + std::string daemon_name; + std::string service_name; // optional; otherwise infer from entity type + + bool service_daemon = false; + std::map daemon_metadata; + std::map daemon_status; + + bool need_metadata_update = false; + + void decode_payload() override + { + using ceph::decode; + auto p = payload.cbegin(); + decode(daemon_name, p); + if (header.version >= 2) { + decode(service_name, p); + decode(service_daemon, p); + if (service_daemon) { + decode(daemon_metadata, p); + decode(daemon_status, p); + } + } + if (header.version >= 3) { + decode(need_metadata_update, p); + } + } + + void encode_payload(uint64_t features) override { + using ceph::encode; + encode(daemon_name, payload); + encode(service_name, payload); + encode(service_daemon, payload); + if (service_daemon) { + encode(daemon_metadata, payload); + encode(daemon_status, payload); + } + encode(need_metadata_update, payload); + } + + std::string_view get_type_name() const override { return "mgrupdate"; } + void print(std::ostream& out) const override { + out << get_type_name() << "("; + if (service_name.length()) { + out << service_name; + } else { + out << ceph_entity_type_name(get_source().type()); + } + out << "." << daemon_name; + if (service_daemon) { + out << " daemon"; + } + out << ")"; + } + +private: + MMgrUpdate() + : Message{MSG_MGR_UPDATE, HEAD_VERSION, COMPAT_VERSION} + {} + using RefCountedObject::put; + using RefCountedObject::get; + template + friend boost::intrusive_ptr ceph::make_message(Args&&... args); +}; + +#endif + diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc index 39647ff796a49..03a5867a33821 100644 --- a/src/mgr/DaemonServer.cc +++ b/src/mgr/DaemonServer.cc @@ -27,6 +27,7 @@ #include "mon/MonCommand.h" #include "messages/MMgrOpen.h" +#include "messages/MMgrUpdate.h" #include "messages/MMgrClose.h" #include "messages/MMgrConfigure.h" #include "messages/MMonMgrReport.h" @@ -267,6 +268,8 @@ bool DaemonServer::ms_dispatch2(const ref_t& m) return handle_report(ref_cast(m)); case MSG_MGR_OPEN: return handle_open(ref_cast(m)); + case MSG_MGR_UPDATE: + return handle_update(ref_cast(m)); case MSG_MGR_CLOSE: return handle_close(ref_cast(m)); case MSG_COMMAND: @@ -531,6 +534,49 @@ bool DaemonServer::handle_open(const ref_t& m) return true; } +bool DaemonServer::handle_update(const ref_t& m) +{ + DaemonKey key; + if (!m->service_name.empty()) { + key.type = m->service_name; + } else { + key.type = ceph_entity_type_name(m->get_connection()->get_peer_type()); + } + key.name = m->daemon_name; + + dout(10) << "from " << m->get_connection() << " " << key << dendl; + + if (m->get_connection()->get_peer_type() == entity_name_t::TYPE_CLIENT && + m->service_name.empty()) { + // Clients should not be sending us update request + dout(10) << "rejecting update request from non-daemon client " << m->daemon_name + << dendl; + clog->warn() << "rejecting report from non-daemon client " << m->daemon_name + << " at " << m->get_connection()->get_peer_addrs(); + m->get_connection()->mark_down(); + return true; + } + + + { + std::unique_lock locker(lock); + + DaemonStatePtr daemon; + // Look up the DaemonState + if (daemon_state.exists(key)) { + dout(20) << "updating existing DaemonState for " << key << dendl; + + daemon = daemon_state.get(key); + if (m->need_metadata_update == true && + !m->daemon_metadata.empty()) { + daemon_state.update_metadata(daemon, m->daemon_metadata); + } + } + } + + return true; +} + bool DaemonServer::handle_close(const ref_t& m) { std::lock_guard l(lock); diff --git a/src/mgr/DaemonServer.h b/src/mgr/DaemonServer.h index d719a96a7bcc3..ff98356807ad9 100644 --- a/src/mgr/DaemonServer.h +++ b/src/mgr/DaemonServer.h @@ -36,6 +36,7 @@ class MMgrReport; class MMgrOpen; +class MMgrUpdate; class MMgrClose; class MMonMgrReport; class MCommand; @@ -275,6 +276,7 @@ public: void fetch_missing_metadata(const DaemonKey& key, const entity_addr_t& addr); bool handle_open(const ceph::ref_t& m); + bool handle_update(const ceph::ref_t& m); bool handle_close(const ceph::ref_t& m); bool handle_report(const ceph::ref_t& m); bool handle_command(const ceph::ref_t& m); diff --git a/src/mgr/MgrClient.cc b/src/mgr/MgrClient.cc index 66ac5901e5099..cfb2e99672ec7 100644 --- a/src/mgr/MgrClient.cc +++ b/src/mgr/MgrClient.cc @@ -21,6 +21,7 @@ #include "messages/MMgrMap.h" #include "messages/MMgrReport.h" #include "messages/MMgrOpen.h" +#include "messages/MMgrUpdate.h" #include "messages/MMgrClose.h" #include "messages/MMgrConfigure.h" #include "messages/MCommand.h" @@ -245,6 +246,25 @@ void MgrClient::_send_open() } } +void MgrClient::_send_update() +{ + if (session && session->con) { + auto update = make_message(); + if (!service_name.empty()) { + update->service_name = service_name; + update->daemon_name = daemon_name; + } else { + update->daemon_name = cct->_conf->name.get_id(); + } + if (service_daemon) { + update->service_daemon = service_daemon; + update->daemon_metadata = daemon_metadata; + } + update->need_metadata_update = need_metadata_update; + session->con->send_message2(update); + } +} + bool MgrClient::handle_mgr_map(ref_t m) { ceph_assert(ceph_mutex_is_locked_by_me(lock)); @@ -566,6 +586,31 @@ bool MgrClient::handle_command_reply( return true; } +int MgrClient::update_daemon_metadata( + const std::string& service, + const std::string& name, + const std::map& metadata) +{ + std::lock_guard l(lock); + if (service_daemon) { + return -EEXIST; + } + ldout(cct,1) << service << "." << name << " metadata " << metadata << dendl; + service_daemon = true; + service_name = service; + daemon_name = name; + daemon_metadata = metadata; + daemon_dirty_status = true; + + if (need_metadata_update == true && + !daemon_metadata.empty()) { + _send_update(); + need_metadata_update = false; + } + + return 0; +} + int MgrClient::service_daemon_register( const std::string& service, const std::string& name, diff --git a/src/mgr/MgrClient.h b/src/mgr/MgrClient.h index f09ba91b712c2..a48ae163e18a8 100644 --- a/src/mgr/MgrClient.h +++ b/src/mgr/MgrClient.h @@ -94,6 +94,7 @@ protected: bool service_daemon = false; bool daemon_dirty_status = false; bool task_dirty_status = false; + bool need_metadata_update = true; std::string service_name, daemon_name; std::map daemon_metadata; std::map daemon_status; @@ -102,6 +103,7 @@ protected: void reconnect(); void _send_open(); + void _send_update(); // In pre-luminous clusters, the ceph-mgr service is absent or optional, // so we must not block in start_command waiting for it. @@ -157,6 +159,10 @@ public: ceph::buffer::list *outbl, std::string *outs, Context *onfinish); + int update_daemon_metadata( + const std::string& service, + const std::string& name, + const std::map& metadata); int service_daemon_register( const std::string& service, const std::string& name, diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index ac6d6632c39a3..b9fc428d3cb32 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -2966,6 +2966,19 @@ void Monitor::log_health( } } +void Monitor::update_pending_metadata() +{ + Metadata metadata; + collect_metadata(&metadata); + size_t version_size = mon_metadata[rank]["ceph_version_short"].size(); + const std::string current_version = mon_metadata[rank]["ceph_version_short"]; + const std::string pending_version = metadata["ceph_version_short"]; + + if (current_version.compare(0, version_size, pending_version) < 0) { + mgr_client.update_daemon_metadata("mon", name, metadata); + } +} + void Monitor::get_cluster_status(stringstream &ss, Formatter *f, MonSession *session) { diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index 454daf63c6c75..575fef0a9c886 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -796,6 +796,8 @@ public: const health_check_map_t& previous, MonitorDBStore::TransactionRef t); + void update_pending_metadata(); + protected: class HealthCheckLogStatus { diff --git a/src/mon/MonmapMonitor.cc b/src/mon/MonmapMonitor.cc index f9792ed575591..6c5a9ce5736a7 100644 --- a/src/mon/MonmapMonitor.cc +++ b/src/mon/MonmapMonitor.cc @@ -248,6 +248,8 @@ void MonmapMonitor::on_active() apply_mon_features(mon.get_quorum_mon_features(), mon.quorum_min_mon_release); + + mon.update_pending_metadata(); } bool MonmapMonitor::preprocess_query(MonOpRequestRef op) diff --git a/src/msg/Message.cc b/src/msg/Message.cc index 6c57d355bdf49..266eb7676786b 100644 --- a/src/msg/Message.cc +++ b/src/msg/Message.cc @@ -190,6 +190,7 @@ #include "messages/MMgrDigest.h" #include "messages/MMgrReport.h" #include "messages/MMgrOpen.h" +#include "messages/MMgrUpdate.h" #include "messages/MMgrClose.h" #include "messages/MMgrConfigure.h" #include "messages/MMonMgrReport.h" @@ -893,6 +894,10 @@ Message *decode_message(CephContext *cct, m = make_message(); break; + case MSG_MGR_UPDATE: + m = make_message(); + break; + case MSG_MGR_CLOSE: m = make_message(); break; diff --git a/src/msg/Message.h b/src/msg/Message.h index 362ae5ec6089d..a7aff1e27a603 100644 --- a/src/msg/Message.h +++ b/src/msg/Message.h @@ -235,6 +235,9 @@ #define MSG_MGR_COMMAND 0x709 #define MSG_MGR_COMMAND_REPLY 0x70a +// *** ceph-mgr <-> MON daemons *** +#define MSG_MGR_UPDATE 0x70b + // ====================================================== // abstract Message class diff --git a/src/msg/MessageRef.h b/src/msg/MessageRef.h index cd5b2dd741a24..ef7a56a44d942 100644 --- a/src/msg/MessageRef.h +++ b/src/msg/MessageRef.h @@ -101,6 +101,7 @@ class MMgrConfigure; class MMgrDigest; class MMgrMap; class MMgrOpen; +class MMgrUpdate; class MMgrReport; class MMonCommandAck; class MMonCommand; -- 2.39.5