From 249d7ef76c95b4fa5fbee93a20bbbfc7d7a40d2c Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Tue, 2 Jul 2019 04:16:11 -0400
Subject: [PATCH] mds: periodically sent mds scrub status to ceph manager

Signed-off-by: Venky Shankar <vshankar@redhat.com>
(cherry picked from commit 625dffe65c0f8001b3b6ca6d0b12732a1a103849)

 Conflicts:
	src/mds/MDSRank.cc
---
 src/common/options.cc |  5 +++++
 src/mds/MDSDaemon.cc  |  2 +-
 src/mds/MDSRank.cc    | 50 +++++++++++++++++++++++++++++++++++++++++--
 src/mds/MDSRank.h     | 11 ++++++++++
 src/mds/ScrubStack.cc | 50 +++++++++++++++++++++++++++++++++++++++++++
 src/mds/ScrubStack.h  |  6 ++++++
 6 files changed, 121 insertions(+), 3 deletions(-)

diff --git a/src/common/options.cc b/src/common/options.cc
index e0018f8249c04..304135391320f 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -8168,6 +8168,11 @@ std::vector<Option> get_mds_options() {
      .set_flag(Option::FLAG_RUNTIME)
      .set_description("max snapshots per directory")
      .set_long_description("maximum number of snapshots that can be created per directory"),
+
+    Option("mds_task_status_update_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
+     .set_default(2.0)
+     .set_description("task status update interval to manager")
+     .set_long_description("interval (in seconds) for sending mds task status to ceph manager"),
   });
 }
 
diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index f42d031f7326f..e07693abc7287 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -875,7 +875,7 @@ void MDSDaemon::handle_mds_map(const MMDSMap::const_ref &m)
     // Did I previously not hold a rank?  Initialize!
     if (mds_rank == NULL) {
       mds_rank = new MDSRankDispatcher(whoami, mds_lock, clog,
-          timer, beacon, mdsmap, messenger, monc,
+          timer, beacon, mdsmap, messenger, monc, &mgrc,
           new FunctionContext([this](int r){respawn();}),
           new FunctionContext([this](int r){suicide();}));
       dout(10) <<  __func__ << ": initializing MDS rank "
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 1d3306f52cae1..967553d798c12 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -21,6 +21,8 @@
 #include "messages/MMDSLoadTargets.h"
 #include "messages/MMDSTableRequest.h"
 
+#include "mgr/MgrClient.h"
+
 #include "MDSDaemon.h"
 #include "MDSMap.h"
 #include "SnapClient.h"
@@ -481,6 +483,7 @@ MDSRank::MDSRank(
     std::unique_ptr<MDSMap>& mdsmap_,
     Messenger *msgr,
     MonClient *monc_,
+    MgrClient *mgrc,
     Context *respawn_hook_,
     Context *suicide_hook_)
   :
@@ -517,7 +520,7 @@ MDSRank::MDSRank(
     hb(NULL), last_tid(0), osd_epoch_barrier(0), beacon(beacon_),
     mds_slow_req_count(0),
     last_client_mdsmap_bcast(0),
-    messenger(msgr), monc(monc_),
+    messenger(msgr), monc(monc_), mgrc(mgrc),
     respawn_hook(respawn_hook_),
     suicide_hook(suicide_hook_),
     standby_replaying(false),
@@ -548,6 +551,15 @@ MDSRank::MDSRank(
                                          cct->_conf->mds_op_log_threshold);
   op_tracker.set_history_size_and_duration(cct->_conf->mds_op_history_size,
                                            cct->_conf->mds_op_history_duration);
+
+  std::string rank_str = stringify(get_nodeid());
+  std::map<std::string, std::string> service_metadata = {{"rank", rank_str}};
+  int r = mgrc->service_daemon_register("mds", rank_str, service_metadata);
+  if (r < 0) {
+    derr << ": failed to register with manager for service status update" << dendl;
+  }
+
+  schedule_update_timer_task();
 }
 
 MDSRank::~MDSRank()
@@ -3512,10 +3524,11 @@ MDSRankDispatcher::MDSRankDispatcher(
     std::unique_ptr<MDSMap> &mdsmap_,
     Messenger *msgr,
     MonClient *monc_,
+    MgrClient *mgrc,
     Context *respawn_hook_,
     Context *suicide_hook_)
   : MDSRank(whoami_, mds_lock_, clog_, timer_, beacon_, mdsmap_,
-      msgr, monc_, respawn_hook_, suicide_hook_)
+            msgr, monc_, mgrc, respawn_hook_, suicide_hook_)
 {
   g_conf().add_observer(this);
 }
@@ -3742,3 +3755,36 @@ void MDSRankDispatcher::handle_conf_change(const ConfigProxy& conf, const std::s
     purge_queue.handle_conf_change(changed, *mdsmap);
   }));
 }
+
+void MDSRank::get_task_status(std::map<std::string, std::string> *status) {
+  dout(20) << __func__ << dendl;
+
+  // scrub summary for now..
+  std::string_view scrub_summary = scrubstack->scrub_summary();
+  status->emplace(SCRUB_STATUS_KEY, std::move(scrub_summary));
+}
+
+void MDSRank::schedule_update_timer_task() {
+  dout(20) << __func__ << dendl;
+
+  timer.add_event_after(g_conf().get_val<double>("mds_task_status_update_interval"),
+                        new FunctionContext([this](int _) {
+                            send_task_status();
+                          }));
+}
+
+void MDSRank::send_task_status() {
+  std::map<std::string, std::string> status;
+  get_task_status(&status);
+
+  if (!status.empty()) {
+    dout(20) << __func__ << ": updating " << status.size() << " status keys" << dendl;
+
+    int r = mgrc->service_daemon_update_task_status(std::move(status));
+    if (r < 0) {
+      derr << ": failed to update service daemon status: " << cpp_strerror(r) << dendl;
+    }
+  }
+
+  schedule_update_timer_task();
+}
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 4bcf9a58c74d1..1ab67f403b353 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -119,6 +119,7 @@ class MDSTableClient;
 class Messenger;
 class Objecter;
 class MonClient;
+class MgrClient;
 class Finisher;
 class ScrubStack;
 class C_MDS_Send_Command_Reply;
@@ -339,6 +340,7 @@ class MDSRank {
         std::unique_ptr<MDSMap> & mdsmap_,
         Messenger *msgr,
         MonClient *monc_,
+        MgrClient *mgrc,
         Context *respawn_hook_,
         Context *suicide_hook_);
 
@@ -505,6 +507,7 @@ class MDSRank {
   protected:
     Messenger    *messenger;
     MonClient    *monc;
+    MgrClient    *mgrc;
 
     Context *respawn_hook;
     Context *suicide_hook;
@@ -577,6 +580,13 @@ class MDSRank {
 private:
     mono_time starttime = mono_clock::zero();
 
+    // "task" string that gets displayed in ceph status
+    inline static const std::string SCRUB_STATUS_KEY = "scrub status";
+
+    void get_task_status(std::map<std::string, std::string> *status);
+    void schedule_update_timer_task();
+    void send_task_status();
+
 protected:
   Context *create_async_exec_context(C_ExecAndReply *ctx);
 };
@@ -653,6 +663,7 @@ public:
       std::unique_ptr<MDSMap> &mdsmap_,
       Messenger *msgr,
       MonClient *monc_,
+      MgrClient *mgrc,
       Context *respawn_hook_,
       Context *suicide_hook_);
 };
diff --git a/src/mds/ScrubStack.cc b/src/mds/ScrubStack.cc
index c36f932fda4a5..818efce977676 100644
--- a/src/mds/ScrubStack.cc
+++ b/src/mds/ScrubStack.cc
@@ -530,6 +530,56 @@ bool ScrubStack::scrub_in_transition_state() {
   return false;
 }
 
+std::string_view ScrubStack::scrub_summary() {
+  ceph_assert(ceph_mutex_is_locked_by_me(mdcache->mds->mds_lock));
+
+  bool have_more = false;
+  CachedStackStringStream cs;
+
+  if (state == STATE_IDLE) {
+    return "idle";
+  }
+
+  if (state == STATE_RUNNING) {
+    if (clear_inode_stack) {
+      *cs << "aborting";
+    } else {
+      *cs << "active";
+    }
+  } else {
+    if (state == STATE_PAUSING) {
+      have_more = true;
+      *cs << "pausing";
+    } else if (state == STATE_PAUSED) {
+      have_more = true;
+      *cs << "paused";
+    }
+
+    if (clear_inode_stack) {
+      if (have_more) {
+        *cs << "+";
+      }
+      *cs << "aborting";
+    }
+  }
+
+  if (!scrub_origins.empty()) {
+    *cs << " [paths:";
+    for (auto inode = scrub_origins.begin(); inode != scrub_origins.end(); ++inode) {
+      if (inode != scrub_origins.begin()) {
+        *cs << ",";
+      }
+
+      std::string path;
+      (*inode)->make_path_string(path, true);
+      *cs << (path.empty() ? "/" : path.c_str());
+    }
+    *cs << "]";
+  }
+
+  return cs->strv();
+}
+
 void ScrubStack::scrub_status(Formatter *f) {
   ceph_assert(mdcache->mds->mds_lock.is_locked_by_me());
 
diff --git a/src/mds/ScrubStack.h b/src/mds/ScrubStack.h
index fb3ececd437cc..8ae674ebec4e3 100644
--- a/src/mds/ScrubStack.h
+++ b/src/mds/ScrubStack.h
@@ -128,6 +128,12 @@ public:
 
   bool is_scrubbing() const { return !inode_stack.empty(); }
 
+  /**
+   * Get a high level scrub status summary such as current scrub state
+   * and scrub paths.
+   */
+  std::string_view scrub_summary();
+
 private:
   // scrub abort is _not_ a state, rather it's an operation that's
   // performed after in-progress scrubs are finished.
-- 
2.39.5