mgr, mon: Keep upto date metadata with mgr for MONs

The mgr updates mon metadata through handle_mon_map which
gets triggered when MONs were removed/added from/to cluster or
the active mgr is restarted or mgr failsover.
We could have handled metadata update through MMgrOpen or
early MMgrReport messages but these are sent before monitor
electin completes and lead monitor updates pending metadata
in monstore. Instead of relying on fetching mon metadata using
'ceph mon metadata <id>' command, explicitly send metadata
update request with mon metadata to mgr.

Fixes: https://tracker.ceph.com/issues/55088
Resolves: rhbz#2121265

Signed-off-by: Prashant D <pdhange@redhat.com>
(cherry picked from commit 1a065043b964f8c014ebb5bc890a243c398ff07c)

diff --git a/src/messages/MMgrUpdate.h b/src/messages/MMgrUpdate.h
new file mode 100644 (file)
index 0000000..71a528d
--- /dev/null
+++ b/src/messages/MMgrUpdate.h
@@ -0,0 +1,92 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2022 Prashant D <pdhange@redhat.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+
+#ifndef CEPH_MMGRUPDATE_H_
+#define CEPH_MMGRUPDATE_H_
+
+#include "msg/Message.h"
+
+class MMgrUpdate : public Message {
+private:
+  static constexpr int HEAD_VERSION = 3;
+  static constexpr int COMPAT_VERSION = 1;
+
+public:
+
+  std::string daemon_name;
+  std::string service_name;  // optional; otherwise infer from entity type
+
+  bool service_daemon = false;
+  std::map<std::string,std::string> daemon_metadata;
+  std::map<std::string,std::string> daemon_status;
+
+  bool need_metadata_update = false;
+
+  void decode_payload() override
+  {
+    using ceph::decode;
+    auto p = payload.cbegin();
+    decode(daemon_name, p);
+    if (header.version >= 2) {
+      decode(service_name, p);
+      decode(service_daemon, p);
+      if (service_daemon) {
+       decode(daemon_metadata, p);
+       decode(daemon_status, p);
+      }
+    }
+    if (header.version >= 3) {
+      decode(need_metadata_update, p);
+    }
+  }
+
+  void encode_payload(uint64_t features) override {
+    using ceph::encode;
+    encode(daemon_name, payload);
+    encode(service_name, payload);
+    encode(service_daemon, payload);
+    if (service_daemon) {
+      encode(daemon_metadata, payload);
+      encode(daemon_status, payload);
+    }
+    encode(need_metadata_update, payload);
+  }
+
+  std::string_view get_type_name() const override { return "mgrupdate"; }
+  void print(std::ostream& out) const override {
+    out << get_type_name() << "(";
+    if (service_name.length()) {
+      out << service_name;
+    } else {
+      out << ceph_entity_type_name(get_source().type());
+    }
+    out << "." << daemon_name;
+    if (service_daemon) {
+      out << " daemon";
+    }
+    out << ")";
+  }
+
+private:
+  MMgrUpdate()
+    : Message{MSG_MGR_UPDATE, HEAD_VERSION, COMPAT_VERSION}
+  {}
+  using RefCountedObject::put;
+  using RefCountedObject::get;
+  template<class T, typename... Args>
+  friend boost::intrusive_ptr<T> ceph::make_message(Args&&... args);
+};
+
+#endif
+

diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc
index 39647ff796a495118a5cd3c9281730cd05ca311f..03a5867a3382198e5dc95f0dea718a7f03dee229 100644 (file)
--- a/src/mgr/DaemonServer.cc
+++ b/src/mgr/DaemonServer.cc
@@ -27,6 +27,7 @@
 #include "mon/MonCommand.h"
 
 #include "messages/MMgrOpen.h"
+#include "messages/MMgrUpdate.h"
 #include "messages/MMgrClose.h"
 #include "messages/MMgrConfigure.h"
 #include "messages/MMonMgrReport.h"
@@ -267,6 +268,8 @@ bool DaemonServer::ms_dispatch2(const ref_t<Message>& m)
       return handle_report(ref_cast<MMgrReport>(m));
     case MSG_MGR_OPEN:
       return handle_open(ref_cast<MMgrOpen>(m));
+    case MSG_MGR_UPDATE:
+      return handle_update(ref_cast<MMgrUpdate>(m));
     case MSG_MGR_CLOSE:
       return handle_close(ref_cast<MMgrClose>(m));
     case MSG_COMMAND:
@@ -531,6 +534,49 @@ bool DaemonServer::handle_open(const ref_t<MMgrOpen>& m)
   return true;
 }
 
+bool DaemonServer::handle_update(const ref_t<MMgrUpdate>& m)
+{
+  DaemonKey key;
+  if (!m->service_name.empty()) {
+    key.type = m->service_name;
+  } else {
+    key.type = ceph_entity_type_name(m->get_connection()->get_peer_type());
+  }
+  key.name = m->daemon_name;
+
+  dout(10) << "from " << m->get_connection() << " " << key << dendl;
+
+  if (m->get_connection()->get_peer_type() == entity_name_t::TYPE_CLIENT &&
+      m->service_name.empty()) {
+    // Clients should not be sending us update request
+    dout(10) << "rejecting update request from non-daemon client " << m->daemon_name
+            << dendl;
+    clog->warn() << "rejecting report from non-daemon client " << m->daemon_name
+                << " at " << m->get_connection()->get_peer_addrs();
+    m->get_connection()->mark_down();
+    return true;
+  }
+
+
+  {
+    std::unique_lock locker(lock);
+
+    DaemonStatePtr daemon;
+    // Look up the DaemonState
+    if (daemon_state.exists(key)) {
+      dout(20) << "updating existing DaemonState for " << key << dendl;
+
+      daemon = daemon_state.get(key);
+      if (m->need_metadata_update == true &&
+          !m->daemon_metadata.empty()) {
+        daemon_state.update_metadata(daemon, m->daemon_metadata);
+      }
+    }
+  }
+
+  return true;
+}
+
 bool DaemonServer::handle_close(const ref_t<MMgrClose>& m)
 {
   std::lock_guard l(lock);

diff --git a/src/mgr/DaemonServer.h b/src/mgr/DaemonServer.h
index d719a96a7bcc36cd3d051b839063d54bc30e4bf7..ff98356807ad976c03ca28d3a0f5927bad83b423 100644 (file)
--- a/src/mgr/DaemonServer.h
+++ b/src/mgr/DaemonServer.h
@@ -36,6 +36,7 @@
 
 class MMgrReport;
 class MMgrOpen;
+class MMgrUpdate;
 class MMgrClose;
 class MMonMgrReport;
 class MCommand;
@@ -275,6 +276,7 @@ public:
 
   void fetch_missing_metadata(const DaemonKey& key, const entity_addr_t& addr);
   bool handle_open(const ceph::ref_t<MMgrOpen>& m);
+  bool handle_update(const ceph::ref_t<MMgrUpdate>& m);
   bool handle_close(const ceph::ref_t<MMgrClose>& m);
   bool handle_report(const ceph::ref_t<MMgrReport>& m);
   bool handle_command(const ceph::ref_t<MCommand>& m);

diff --git a/src/mgr/MgrClient.cc b/src/mgr/MgrClient.cc
index 66ac5901e50998e122a91521b923750dfe2182cb..cfb2e99672ec7bfe2066772de5efc63e5162bccd 100644 (file)
--- a/src/mgr/MgrClient.cc
+++ b/src/mgr/MgrClient.cc
@@ -21,6 +21,7 @@
 #include "messages/MMgrMap.h"
 #include "messages/MMgrReport.h"
 #include "messages/MMgrOpen.h"
+#include "messages/MMgrUpdate.h"
 #include "messages/MMgrClose.h"
 #include "messages/MMgrConfigure.h"
 #include "messages/MCommand.h"
@@ -245,6 +246,25 @@ void MgrClient::_send_open()
   }
 }
 
+void MgrClient::_send_update()
+{
+  if (session && session->con) {
+    auto update = make_message<MMgrUpdate>();
+    if (!service_name.empty()) {
+      update->service_name = service_name;
+      update->daemon_name = daemon_name;
+    } else {
+      update->daemon_name = cct->_conf->name.get_id();
+    }
+    if (service_daemon) {
+      update->service_daemon = service_daemon;
+      update->daemon_metadata = daemon_metadata;
+    }
+    update->need_metadata_update = need_metadata_update;
+    session->con->send_message2(update);
+  }
+}
+
 bool MgrClient::handle_mgr_map(ref_t<MMgrMap> m)
 {
   ceph_assert(ceph_mutex_is_locked_by_me(lock));
@@ -566,6 +586,31 @@ bool MgrClient::handle_command_reply(
   return true;
 }
 
+int MgrClient::update_daemon_metadata(
+  const std::string& service,
+  const std::string& name,
+  const std::map<std::string,std::string>& metadata)
+{
+  std::lock_guard l(lock);
+  if (service_daemon) {
+    return -EEXIST;
+  }
+  ldout(cct,1) << service << "." << name << " metadata " << metadata << dendl;
+  service_daemon = true;
+  service_name = service;
+  daemon_name = name;
+  daemon_metadata = metadata;
+  daemon_dirty_status = true;
+
+  if (need_metadata_update == true &&
+      !daemon_metadata.empty()) {
+    _send_update();
+    need_metadata_update = false;
+  }
+
+  return 0;
+}
+
 int MgrClient::service_daemon_register(
   const std::string& service,
   const std::string& name,

diff --git a/src/mgr/MgrClient.h b/src/mgr/MgrClient.h
index f09ba91b712c2ef4032acc81e581445280ee708d..a48ae163e18a8da011efeb81729d7022dc1e8da3 100644 (file)
--- a/src/mgr/MgrClient.h
+++ b/src/mgr/MgrClient.h
@@ -94,6 +94,7 @@ protected:
   bool service_daemon = false;
   bool daemon_dirty_status = false;
   bool task_dirty_status = false;
+  bool need_metadata_update = true;
   std::string service_name, daemon_name;
   std::map<std::string,std::string> daemon_metadata;
   std::map<std::string,std::string> daemon_status;
@@ -102,6 +103,7 @@ protected:
 
   void reconnect();
   void _send_open();
+  void _send_update();
 
   // In pre-luminous clusters, the ceph-mgr service is absent or optional,
   // so we must not block in start_command waiting for it.
@@ -157,6 +159,10 @@ public:
     ceph::buffer::list *outbl, std::string *outs,
     Context *onfinish);
 
+  int update_daemon_metadata(
+    const std::string& service,
+    const std::string& name,
+    const std::map<std::string,std::string>& metadata);
   int service_daemon_register(
     const std::string& service,
     const std::string& name,

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index ac6d6632c39a3a1399915aaf66b86e9d1d864e8a..b9fc428d3cb32469e497805d150e473ef42b1579 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -2966,6 +2966,19 @@ void Monitor::log_health(
   }
 }
 
+void Monitor::update_pending_metadata()
+{
+  Metadata metadata;
+  collect_metadata(&metadata);
+  size_t version_size = mon_metadata[rank]["ceph_version_short"].size();
+  const std::string current_version = mon_metadata[rank]["ceph_version_short"];
+  const std::string pending_version = metadata["ceph_version_short"];
+
+  if (current_version.compare(0, version_size, pending_version) < 0) {
+    mgr_client.update_daemon_metadata("mon", name, metadata);
+  }
+}
+
 void Monitor::get_cluster_status(stringstream &ss, Formatter *f,
                                 MonSession *session)
 {

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 454daf63c6c7583128633ad8aa511353c98b4e94..575fef0a9c886059241b9fca30725a075a2d4445 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -796,6 +796,8 @@ public:
     const health_check_map_t& previous,
     MonitorDBStore::TransactionRef t);
 
+  void update_pending_metadata();
+
 protected:
 
   class HealthCheckLogStatus {

diff --git a/src/mon/MonmapMonitor.cc b/src/mon/MonmapMonitor.cc
index f9792ed575591b6a11bbdff551ce6422e1a0cf8e..6c5a9ce5736a7fa1e12593d3450d1f517fd7573f 100644 (file)
--- a/src/mon/MonmapMonitor.cc
+++ b/src/mon/MonmapMonitor.cc
@@ -248,6 +248,8 @@ void MonmapMonitor::on_active()
 
   apply_mon_features(mon.get_quorum_mon_features(),
                     mon.quorum_min_mon_release);
+
+  mon.update_pending_metadata();
 }
 
 bool MonmapMonitor::preprocess_query(MonOpRequestRef op)

diff --git a/src/msg/Message.cc b/src/msg/Message.cc
index 6c57d355bdf49f95f90a2f624a3a705290c2cd6d..266eb7676786b85ea7481855fa6d8bd94c600a56 100644 (file)
--- a/src/msg/Message.cc
+++ b/src/msg/Message.cc
@@ -190,6 +190,7 @@
 #include "messages/MMgrDigest.h"
 #include "messages/MMgrReport.h"
 #include "messages/MMgrOpen.h"
+#include "messages/MMgrUpdate.h"
 #include "messages/MMgrClose.h"
 #include "messages/MMgrConfigure.h"
 #include "messages/MMonMgrReport.h"
@@ -893,6 +894,10 @@ Message *decode_message(CephContext *cct,
     m = make_message<MMgrOpen>();
     break;
 
+  case MSG_MGR_UPDATE:
+    m = make_message<MMgrUpdate>();
+    break;
+
   case MSG_MGR_CLOSE:
     m = make_message<MMgrClose>();
     break;

diff --git a/src/msg/Message.h b/src/msg/Message.h
index 362ae5ec6089d9c8eeb988aeced8b375ea5a248e..a7aff1e27a603fbc4f3274ad47a447a8ea27d246 100644 (file)
--- a/src/msg/Message.h
+++ b/src/msg/Message.h
@@ -235,6 +235,9 @@
 #define MSG_MGR_COMMAND           0x709
 #define MSG_MGR_COMMAND_REPLY     0x70a
 
+// *** ceph-mgr <-> MON daemons ***
+#define MSG_MGR_UPDATE     0x70b
+
 // ======================================================
 
 // abstract Message class

diff --git a/src/msg/MessageRef.h b/src/msg/MessageRef.h
index cd5b2dd741a247cfacbdad3aca35b5bcace17b6d..ef7a56a44d942cd1059348f9aed73ce15582a020 100644 (file)
--- a/src/msg/MessageRef.h
+++ b/src/msg/MessageRef.h
@@ -101,6 +101,7 @@ class MMgrConfigure;
 class MMgrDigest;
 class MMgrMap;
 class MMgrOpen;
+class MMgrUpdate;
 class MMgrReport;
 class MMonCommandAck;
 class MMonCommand;