client: periodically forward metrics to session MDSs

Fixes: https://tracker.ceph.com/issues/26996
Signed-off-by: Venky Shankar <vshankar@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index d9c0bb0d270830b5792fee04be37536d4b1a797f..31853d4b074f121032230d96c03a3c840821d70d 100755 (executable)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -61,6 +61,7 @@
 #include "messages/MClientRequestForward.h"
 #include "messages/MClientSession.h"
 #include "messages/MClientSnap.h"
+#include "messages/MClientMetrics.h"
 #include "messages/MCommandReply.h"
 #include "messages/MFSMap.h"
 #include "messages/MFSMapUser.h"
@@ -2058,6 +2059,7 @@ MetaSession *Client::_open_mds_session(mds_rank_t mds)
   auto m = make_message<MClientSession>(CEPH_SESSION_REQUEST_OPEN);
   m->metadata = metadata;
   m->supported_features = feature_bitset_t(CEPHFS_FEATURES_CLIENT_SUPPORTED);
+  m->metric_spec = feature_bitset_t(CEPHFS_METRIC_FEATURES_ALL);
   session->con->send_message2(std::move(m));
   return session;
 }
@@ -6359,6 +6361,8 @@ void Client::tick()
     check_caps(in, CHECK_CAPS_NODELAY);
   }
 
+  collect_and_send_metrics();
+
   trim_cache(true);
 
   if (blacklisted && mounted &&
@@ -6372,6 +6376,54 @@ void Client::tick()
   }
 }
 
+void Client::collect_and_send_metrics() {
+  ldout(cct, 20) << __func__ << dendl;
+
+  ceph_assert(ceph_mutex_is_locked_by_me(client_lock));
+
+  // right now, we only track and send global metrics. its sufficient
+  // to send these metrics to MDS rank0.
+  collect_and_send_global_metrics();
+}
+
+void Client::collect_and_send_global_metrics() {
+  ldout(cct, 20) << __func__ << dendl;
+  ceph_assert(ceph_mutex_is_locked_by_me(client_lock));
+
+  if (!have_open_session((mds_rank_t)0)) {
+    ldout(cct, 5) << __func__ << ": no session with rank=0 -- not sending metric"
+                  << dendl;
+    return;
+  }
+  auto session = _get_or_open_mds_session((mds_rank_t)0);
+  if (!session->mds_features.test(CEPHFS_FEATURE_METRIC_COLLECT)) {
+    ldout(cct, 5) << __func__ << ": rank=0 does not support metrics" << dendl;
+    return;
+  }
+
+  ClientMetricMessage metric;
+  std::vector<ClientMetricMessage> message;
+
+  // read latency
+  metric = ClientMetricMessage(ReadLatencyPayload(logger->tget(l_c_read)));
+  message.push_back(metric);
+
+  // write latency
+  metric = ClientMetricMessage(WriteLatencyPayload(logger->tget(l_c_wrlat)));
+  message.push_back(metric);
+
+  // metadata latency
+  metric = ClientMetricMessage(MetadataLatencyPayload(logger->tget(l_c_lat)));
+  message.push_back(metric);
+
+  // cap hit ratio -- nr_caps is unused right now
+  auto [cap_hits, cap_misses] = get_cap_hit_rates();
+  metric = ClientMetricMessage(CapInfoPayload(cap_hits, cap_misses, 0));
+  message.push_back(metric);
+
+  session->con->send_message2(make_message<MClientMetrics>(std::move(message)));
+}
+
 void Client::renew_caps()
 {
   ldout(cct, 10) << "renew_caps()" << dendl;

diff --git a/src/client/Client.h b/src/client/Client.h
index 002357a3ee596de5ac615d6ae23fcc3e358ca061..fa53b287d6ae1142695acedfdc732363af2b1695 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -30,6 +30,7 @@
 #include "include/types.h"
 #include "include/unordered_map.h"
 #include "include/unordered_set.h"
+#include "include/cephfs/metrics/Types.h"
 #include "mds/mdstypes.h"
 #include "msg/Dispatcher.h"
 #include "msg/MessageRef.h"
@@ -1190,6 +1191,8 @@ private:
   int _lookup_ino(inodeno_t ino, const UserPerm& perms, Inode **inode=NULL);
   bool _ll_forget(Inode *in, uint64_t count);
 
+  void collect_and_send_metrics();
+  void collect_and_send_global_metrics();
 
   uint32_t deleg_timeout = 0;
 

diff --git a/src/mds/cephfs_features.h b/src/mds/cephfs_features.h
index 75309ef5da89f046758f7659d6c06aba156a9a3b..a8610e99e41a31ace9262b26519748ad2aaffc5f 100644 (file)
--- a/src/mds/cephfs_features.h
+++ b/src/mds/cephfs_features.h
@@ -15,6 +15,8 @@
 #ifndef CEPHFS_FEATURES_H
 #define CEPHFS_FEATURES_H
 
+#include "include/cephfs/metrics/Types.h"
+
 class feature_bitset_t;
 namespace ceph {
   class Formatter;
@@ -58,6 +60,13 @@ namespace ceph {
   CEPHFS_FEATURE_METRIC_COLLECT,        \
 }
 
+#define CEPHFS_METRIC_FEATURES_ALL {           \
+    CLIENT_METRIC_TYPE_CAP_INFO,               \
+    CLIENT_METRIC_TYPE_READ_LATENCY,           \
+    CLIENT_METRIC_TYPE_WRITE_LATENCY,          \
+    CLIENT_METRIC_TYPE_METADATA_LATENCY,       \
+}
+
 #define CEPHFS_FEATURES_MDS_SUPPORTED CEPHFS_FEATURES_ALL
 #define CEPHFS_FEATURES_MDS_REQUIRED {}