mgr, mon: Keep upto date metadata with mgr for MONs

The mgr updates mon metadata through handle_mon_map which
gets triggered when MONs were removed/added from/to cluster or
the active mgr is restarted or mgr failsover.
We could have handled metadata update through MMgrOpen or
early MMgrReport messages but these are sent before monitor
electin completes and lead monitor updates pending metadata
in monstore. Instead of relying on fetching mon metadata using
'ceph mon metadata <id>' command, explicitly send metadata
update request with mon metadata to mgr.

Fixes: https://tracker.ceph.com/issues/55088
Signed-off-by: Prashant D <pdhange@redhat.com>
(cherry picked from commit 1a065043b964f8c014ebb5bc890a243c398ff07c)

diff --git a/src/messages/MMgrUpdate.h b/src/messages/MMgrUpdate.h
new file mode 100644 (file)
index 0000000..71a528d
--- /dev/null
+++ b/src/messages/MMgrUpdate.h
@@ -0,0 +1,92 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2022 Prashant D <pdhange@redhat.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+
+#ifndef CEPH_MMGRUPDATE_H_
+#define CEPH_MMGRUPDATE_H_
+
+#include "msg/Message.h"
+
+class MMgrUpdate : public Message {
+private:
+  static constexpr int HEAD_VERSION = 3;
+  static constexpr int COMPAT_VERSION = 1;
+
+public:
+
+  std::string daemon_name;
+  std::string service_name;  // optional; otherwise infer from entity type
+
+  bool service_daemon = false;
+  std::map<std::string,std::string> daemon_metadata;
+  std::map<std::string,std::string> daemon_status;
+
+  bool need_metadata_update = false;
+
+  void decode_payload() override
+  {
+    using ceph::decode;
+    auto p = payload.cbegin();
+    decode(daemon_name, p);
+    if (header.version >= 2) {
+      decode(service_name, p);
+      decode(service_daemon, p);
+      if (service_daemon) {
+       decode(daemon_metadata, p);
+       decode(daemon_status, p);
+      }
+    }
+    if (header.version >= 3) {
+      decode(need_metadata_update, p);
+    }
+  }
+
+  void encode_payload(uint64_t features) override {
+    using ceph::encode;
+    encode(daemon_name, payload);
+    encode(service_name, payload);
+    encode(service_daemon, payload);
+    if (service_daemon) {
+      encode(daemon_metadata, payload);
+      encode(daemon_status, payload);
+    }
+    encode(need_metadata_update, payload);
+  }
+
+  std::string_view get_type_name() const override { return "mgrupdate"; }
+  void print(std::ostream& out) const override {
+    out << get_type_name() << "(";
+    if (service_name.length()) {
+      out << service_name;
+    } else {
+      out << ceph_entity_type_name(get_source().type());
+    }
+    out << "." << daemon_name;
+    if (service_daemon) {
+      out << " daemon";
+    }
+    out << ")";
+  }
+
+private:
+  MMgrUpdate()
+    : Message{MSG_MGR_UPDATE, HEAD_VERSION, COMPAT_VERSION}
+  {}
+  using RefCountedObject::put;
+  using RefCountedObject::get;
+  template<class T, typename... Args>
+  friend boost::intrusive_ptr<T> ceph::make_message(Args&&... args);
+};
+
+#endif
+

diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc
index b1a3c5c8e4a5d9f84ef2113c2b00cd46d651c070..1ab00e0afd6f393606b0f1363b2800b37e82ec96 100644 (file)
--- a/src/mgr/DaemonServer.cc
+++ b/src/mgr/DaemonServer.cc
@@ -27,6 +27,7 @@
 #include "mon/MonCommand.h"
 
 #include "messages/MMgrOpen.h"
+#include "messages/MMgrUpdate.h"
 #include "messages/MMgrClose.h"
 #include "messages/MMgrConfigure.h"
 #include "messages/MMonMgrReport.h"
@@ -258,6 +259,8 @@ bool DaemonServer::ms_dispatch2(const ref_t<Message>& m)
       return handle_report(ref_cast<MMgrReport>(m));
     case MSG_MGR_OPEN:
       return handle_open(ref_cast<MMgrOpen>(m));
+    case MSG_MGR_UPDATE:
+      return handle_update(ref_cast<MMgrUpdate>(m));
     case MSG_MGR_CLOSE:
       return handle_close(ref_cast<MMgrClose>(m));
     case MSG_COMMAND:
@@ -522,6 +525,49 @@ bool DaemonServer::handle_open(const ref_t<MMgrOpen>& m)
   return true;
 }
 
+bool DaemonServer::handle_update(const ref_t<MMgrUpdate>& m)
+{
+  DaemonKey key;
+  if (!m->service_name.empty()) {
+    key.type = m->service_name;
+  } else {
+    key.type = ceph_entity_type_name(m->get_connection()->get_peer_type());
+  }
+  key.name = m->daemon_name;
+
+  dout(10) << "from " << m->get_connection() << " " << key << dendl;
+
+  if (m->get_connection()->get_peer_type() == entity_name_t::TYPE_CLIENT &&
+      m->service_name.empty()) {
+    // Clients should not be sending us update request
+    dout(10) << "rejecting update request from non-daemon client " << m->daemon_name
+            << dendl;
+    clog->warn() << "rejecting report from non-daemon client " << m->daemon_name
+                << " at " << m->get_connection()->get_peer_addrs();
+    m->get_connection()->mark_down();
+    return true;
+  }
+
+
+  {
+    std::unique_lock locker(lock);
+
+    DaemonStatePtr daemon;
+    // Look up the DaemonState
+    if (daemon_state.exists(key)) {
+      dout(20) << "updating existing DaemonState for " << key << dendl;
+
+      daemon = daemon_state.get(key);
+      if (m->need_metadata_update == true &&
+          !m->daemon_metadata.empty()) {
+        daemon_state.update_metadata(daemon, m->daemon_metadata);
+      }
+    }
+  }
+
+  return true;
+}
+
 bool DaemonServer::handle_close(const ref_t<MMgrClose>& m)
 {
   std::lock_guard l(lock);

diff --git a/src/mgr/DaemonServer.h b/src/mgr/DaemonServer.h
index 3adcf6ccaf93b8d50af5cd15202b115492e4340a..c746498a56b6f07ce208edfbe3809ec0655965c6 100644 (file)
--- a/src/mgr/DaemonServer.h
+++ b/src/mgr/DaemonServer.h
@@ -36,6 +36,7 @@
 
 class MMgrReport;
 class MMgrOpen;
+class MMgrUpdate;
 class MMgrClose;
 class MMonMgrReport;
 class MCommand;
@@ -275,6 +276,7 @@ public:
 
   void fetch_missing_metadata(const DaemonKey& key, const entity_addr_t& addr);
   bool handle_open(const ceph::ref_t<MMgrOpen>& m);
+  bool handle_update(const ceph::ref_t<MMgrUpdate>& m);
   bool handle_close(const ceph::ref_t<MMgrClose>& m);
   bool handle_report(const ceph::ref_t<MMgrReport>& m);
   bool handle_command(const ceph::ref_t<MCommand>& m);

diff --git a/src/mgr/MgrClient.cc b/src/mgr/MgrClient.cc
index 4ac88dd99d4627073c46137026071f7910e7edc8..626c9941a09dab00bb5d0155b5584c2f72492c5d 100644 (file)
--- a/src/mgr/MgrClient.cc
+++ b/src/mgr/MgrClient.cc
@@ -21,6 +21,7 @@
 #include "messages/MMgrMap.h"
 #include "messages/MMgrReport.h"
 #include "messages/MMgrOpen.h"
+#include "messages/MMgrUpdate.h"
 #include "messages/MMgrClose.h"
 #include "messages/MMgrConfigure.h"
 #include "messages/MCommand.h"
@@ -245,6 +246,25 @@ void MgrClient::_send_open()
   }
 }
 
+void MgrClient::_send_update()
+{
+  if (session && session->con) {
+    auto update = make_message<MMgrUpdate>();
+    if (!service_name.empty()) {
+      update->service_name = service_name;
+      update->daemon_name = daemon_name;
+    } else {
+      update->daemon_name = cct->_conf->name.get_id();
+    }
+    if (service_daemon) {
+      update->service_daemon = service_daemon;
+      update->daemon_metadata = daemon_metadata;
+    }
+    update->need_metadata_update = need_metadata_update;
+    session->con->send_message2(update);
+  }
+}
+
 bool MgrClient::handle_mgr_map(ref_t<MMgrMap> m)
 {
   ceph_assert(ceph_mutex_is_locked_by_me(lock));
@@ -568,6 +588,31 @@ bool MgrClient::handle_command_reply(
   return true;
 }
 
+int MgrClient::update_daemon_metadata(
+  const std::string& service,
+  const std::string& name,
+  const std::map<std::string,std::string>& metadata)
+{
+  std::lock_guard l(lock);
+  if (service_daemon) {
+    return -EEXIST;
+  }
+  ldout(cct,1) << service << "." << name << " metadata " << metadata << dendl;
+  service_daemon = true;
+  service_name = service;
+  daemon_name = name;
+  daemon_metadata = metadata;
+  daemon_dirty_status = true;
+
+  if (need_metadata_update == true &&
+      !daemon_metadata.empty()) {
+    _send_update();
+    need_metadata_update = false;
+  }
+
+  return 0;
+}
+
 int MgrClient::service_daemon_register(
   const std::string& service,
   const std::string& name,

diff --git a/src/mgr/MgrClient.h b/src/mgr/MgrClient.h
index 6bd2b26e11a37c4cc272585807fe9f7b34e0c62c..1668d8da0fbc29fc64894e21b0fa5aefc58ca27b 100644 (file)
--- a/src/mgr/MgrClient.h
+++ b/src/mgr/MgrClient.h
@@ -94,6 +94,7 @@ protected:
   bool service_daemon = false;
   bool daemon_dirty_status = false;
   bool task_dirty_status = false;
+  bool need_metadata_update = true;
   std::string service_name, daemon_name;
   std::map<std::string,std::string> daemon_metadata;
   std::map<std::string,std::string> daemon_status;
@@ -102,6 +103,7 @@ protected:
 
   void reconnect();
   void _send_open();
+  void _send_update();
 
   // In pre-luminous clusters, the ceph-mgr service is absent or optional,
   // so we must not block in start_command waiting for it.
@@ -157,6 +159,10 @@ public:
     ceph::buffer::list *outbl, std::string *outs,
     Context *onfinish);
 
+  int update_daemon_metadata(
+    const std::string& service,
+    const std::string& name,
+    const std::map<std::string,std::string>& metadata);
   int service_daemon_register(
     const std::string& service,
     const std::string& name,

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index ab9d49acce224684c369742e3740880e509ccadf..aee9198a936d6fddf6b648fac32a593d5526a19f 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -2966,6 +2966,19 @@ void Monitor::log_health(
   }
 }
 
+void Monitor::update_pending_metadata()
+{
+  Metadata metadata;
+  collect_metadata(&metadata);
+  size_t version_size = mon_metadata[rank]["ceph_version_short"].size();
+  const std::string current_version = mon_metadata[rank]["ceph_version_short"];
+  const std::string pending_version = metadata["ceph_version_short"];
+
+  if (current_version.compare(0, version_size, pending_version) < 0) {
+    mgr_client.update_daemon_metadata("mon", name, metadata);
+  }
+}
+
 void Monitor::get_cluster_status(stringstream &ss, Formatter *f,
                                 MonSession *session)
 {

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index e16dc4a323a32770a6417372e3d7d7ab1b9892f3..1f7e64e8592cbbd8e4cff8a2a06fe253b4305723 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -796,6 +796,8 @@ public:
     const health_check_map_t& previous,
     MonitorDBStore::TransactionRef t);
 
+  void update_pending_metadata();
+
 protected:
 
   class HealthCheckLogStatus {

diff --git a/src/mon/MonmapMonitor.cc b/src/mon/MonmapMonitor.cc
index c9dfa3af499b15bf49eca86d0dd74f89a93e6e6f..1ecbea8578c27222bfb0cb8e32665fb247de4ea2 100644 (file)
--- a/src/mon/MonmapMonitor.cc
+++ b/src/mon/MonmapMonitor.cc
@@ -248,6 +248,8 @@ void MonmapMonitor::on_active()
 
   apply_mon_features(mon.get_quorum_mon_features(),
                     mon.quorum_min_mon_release);
+
+  mon.update_pending_metadata();
 }
 
 bool MonmapMonitor::preprocess_query(MonOpRequestRef op)

diff --git a/src/msg/Message.cc b/src/msg/Message.cc
index 6c57d355bdf49f95f90a2f624a3a705290c2cd6d..266eb7676786b85ea7481855fa6d8bd94c600a56 100644 (file)
--- a/src/msg/Message.cc
+++ b/src/msg/Message.cc
@@ -190,6 +190,7 @@
 #include "messages/MMgrDigest.h"
 #include "messages/MMgrReport.h"
 #include "messages/MMgrOpen.h"
+#include "messages/MMgrUpdate.h"
 #include "messages/MMgrClose.h"
 #include "messages/MMgrConfigure.h"
 #include "messages/MMonMgrReport.h"
@@ -893,6 +894,10 @@ Message *decode_message(CephContext *cct,
     m = make_message<MMgrOpen>();
     break;
 
+  case MSG_MGR_UPDATE:
+    m = make_message<MMgrUpdate>();
+    break;
+
   case MSG_MGR_CLOSE:
     m = make_message<MMgrClose>();
     break;

diff --git a/src/msg/Message.h b/src/msg/Message.h
index 8e1c4f967f05942855a6f34e9f1135a8470eb190..c3b47eee20b07b67be80b5c0d8f23586d5c9e87b 100644 (file)
--- a/src/msg/Message.h
+++ b/src/msg/Message.h
@@ -235,6 +235,9 @@
 #define MSG_MGR_COMMAND           0x709
 #define MSG_MGR_COMMAND_REPLY     0x70a
 
+// *** ceph-mgr <-> MON daemons ***
+#define MSG_MGR_UPDATE     0x70b
+
 // ======================================================
 
 // abstract Message class

diff --git a/src/msg/MessageRef.h b/src/msg/MessageRef.h
index 2c11aced52b34f559503726120f1aec6cee1f5ee..6bed38311c1480b466fed38af3029b9cba8836a1 100644 (file)
--- a/src/msg/MessageRef.h
+++ b/src/msg/MessageRef.h
@@ -97,6 +97,7 @@ class MMgrConfigure;
 class MMgrDigest;
 class MMgrMap;
 class MMgrOpen;
+class MMgrUpdate;
 class MMgrReport;
 class MMonCommandAck;
 class MMonCommand;