mon: DataHealthService: shutdown mon if failed to obtain disk stats

Being unable to run a ::statfs() may be a symptom of something bigger.

We want to cleanly shutdown the monitor ASAP if such thing happens.

Fixes: #4509
Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>

diff --git a/src/mon/DataHealthService.cc b/src/mon/DataHealthService.cc
index d97b458ec0359a4424ce8f667e367e94c4eff5cf..b8462ca72cd69ae74319c4fac80614b49bb301a6 100644 (file)
--- a/src/mon/DataHealthService.cc
+++ b/src/mon/DataHealthService.cc
@@ -102,14 +102,13 @@ void DataHealthService::get_health(Formatter *f,
   f->close_section(); // data_health
 }
 
-void DataHealthService::update_stats()
+int DataHealthService::update_stats()
 {
   struct statfs stbuf;
   int err = ::statfs(g_conf->mon_data.c_str(), &stbuf);
   if (err < 0) {
-    assert(errno != EIO);
-    mon->clog.error() << __func__ << " statfs error: " << cpp_strerror(errno) << "\n";
-    return;
+    derr << __func__ << " statfs error: " << cpp_strerror(errno) << dendl;
+    return -errno;
   }
 
   entity_inst_t our_inst = mon->monmap->get_inst(mon->name);
@@ -123,6 +122,7 @@ void DataHealthService::update_stats()
           << " total " << ours.kb_total << " used " << ours.kb_used << " avail " << ours.kb_avail
           << dendl;
   ours.last_update = ceph_clock_now(g_ceph_context);
+  return 0;
 }
 
 void DataHealthService::share_stats()
@@ -153,7 +153,13 @@ void DataHealthService::service_tick()
 {
   dout(10) << __func__ << dendl;
 
-  update_stats();
+  int err = update_stats();
+  if (err < 0) {
+    derr << "something went wrong obtaining our disk stats: "
+         << cpp_strerror(err) << dendl;
+    force_shutdown();
+    return;
+  }
   if (in_quorum())
     share_stats();
 

diff --git a/src/mon/DataHealthService.h b/src/mon/DataHealthService.h
index d54af792d9c13fdbbee277c03c82d486c3d2c14e..9d4e1e2210dfc6007f0198921211bfe4638f65ca 100644 (file)
--- a/src/mon/DataHealthService.h
+++ b/src/mon/DataHealthService.h
@@ -35,7 +35,7 @@ class DataHealthService :
 {
   map<entity_inst_t,DataStats> stats;
   void handle_tell(MMonHealth *m);
-  void update_stats();
+  int update_stats();
   void share_stats();
 
   void force_shutdown() {