mon, osd: Add detailed full information for now in the mon

Show ceph health doc output in the correct order

Signed-off-by: David Zafman <dzafman@redhat.com>

diff --git a/doc/rados/troubleshooting/troubleshooting-osd.rst b/doc/rados/troubleshooting/troubleshooting-osd.rst
index 651907dfb0589e40e88d59157f20e5a6b4a3026e..fe29f4767f94d4b433268389eef1cfef9dc903ae 100644 (file)
--- a/doc/rados/troubleshooting/troubleshooting-osd.rst
+++ b/doc/rados/troubleshooting/troubleshooting-osd.rst
@@ -222,16 +222,15 @@ lowering the ``mon osd full ratio``, ``mon osd backfillfull ratio``  and
 Full ``ceph-osds`` will be reported by ``ceph health``::
 
        ceph health
-       HEALTH_WARN 1 nearfull osds
-       osd.2 is near full at 85%
+       HEALTH_WARN 1 nearfull osd(s)
 
 Or::
 
-       ceph health
-       HEALTH_ERR 1 nearfull osds, 1 backfillfull osds, 1 full osds
-       osd.2 is near full at 85%
+       ceph health detail
+       HEALTH_ERR 1 full osd(s); 1 backfillfull osd(s); 1 nearfull osd(s)
        osd.3 is full at 97%
        osd.4 is backfill full at 91%
+       osd.2 is near full at 87%
 
 The best way to deal with a full cluster is to add new ``ceph-osds``, allowing
 the cluster to redistribute data to the newly available storage.

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 680e9e3a6fe3597e157343824fec9aba83bb5755..0b059271fbb38ff1a2ba2c56e342e5c1b0b1a33c 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -3402,23 +3402,40 @@ void OSDMonitor::get_health(list<pair<health_status_t,string> >& summary,
        summary.push_back(make_pair(HEALTH_ERR, ss.str()));
       }
 
-      int full, backfill, nearfull;
-      osdmap.count_full_nearfull_osds(&full, &backfill, &nearfull);
-      if (full > 0) {
+      map<int, float> full, backfillfull, nearfull;
+      osdmap.get_full_osd_util(mon->pgmon()->pg_map.osd_stat, &full, &backfillfull, &nearfull);
+      if (full.size()) {
        ostringstream ss;
-       ss << full << " full osd(s)";
+       ss << full.size() << " full osd(s)";
        summary.push_back(make_pair(HEALTH_ERR, ss.str()));
       }
-      if (backfill > 0) {
+      if (backfillfull.size()) {
        ostringstream ss;
-       ss << backfill << " backfillfull osd(s)";
+       ss << backfillfull.size() << " backfillfull osd(s)";
        summary.push_back(make_pair(HEALTH_WARN, ss.str()));
       }
-      if (nearfull > 0) {
+      if (nearfull.size()) {
        ostringstream ss;
-       ss << nearfull << " nearfull osd(s)";
+       ss << nearfull.size() << " nearfull osd(s)";
        summary.push_back(make_pair(HEALTH_WARN, ss.str()));
       }
+      if (detail) {
+        for (auto& i: full) {
+          ostringstream ss;
+          ss << "osd." << i.first << " is full at " << roundf(i.second * 100) << "%";
+         detail->push_back(make_pair(HEALTH_ERR, ss.str()));
+        }
+        for (auto& i: backfillfull) {
+          ostringstream ss;
+          ss << "osd." << i.first << " is backfill full at " << roundf(i.second * 100) << "%";
+         detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+        }
+        for (auto& i: nearfull) {
+          ostringstream ss;
+          ss << "osd." << i.first << " is near full at " << roundf(i.second * 100) << "%";
+         detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+        }
+      }
     }
     // note: we leave it to ceph-mgr to generate details health warnings
     // with actual osd utilizations

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 6d0cbfe0a283e7acf04a5378b8a17109b8cab38c..5035fab931a8827006245a5ad0e382781e998bf6 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -1046,6 +1046,40 @@ void OSDMap::count_full_nearfull_osds(int *full, int *backfill, int *nearfull) c
   }
 }
 
+static bool get_osd_utilization(const ceph::unordered_map<int32_t,osd_stat_t> &osd_stat,
+   int id, int64_t* kb, int64_t* kb_used, int64_t* kb_avail) {
+    auto p = osd_stat.find(id);
+    if (p == osd_stat.end())
+      return false;
+    *kb = p->second.kb;
+    *kb_used = p->second.kb_used;
+    *kb_avail = p->second.kb_avail;
+    return *kb > 0;
+}
+
+void OSDMap::get_full_osd_util(const ceph::unordered_map<int32_t,osd_stat_t> &osd_stat,
+     map<int, float> *full, map<int, float> *backfill, map<int, float> *nearfull) const
+{
+  full->clear();
+  backfill->clear();
+  nearfull->clear();
+  for (int i = 0; i < max_osd; ++i) {
+    if (exists(i) && is_up(i) && is_in(i)) {
+      int64_t kb, kb_used, kb_avail;
+      if (osd_state[i] & CEPH_OSD_FULL) {
+        if (get_osd_utilization(osd_stat, i, &kb, &kb_used, &kb_avail))
+         full->emplace(i, (float)kb_used / (float)kb);
+      } else if (osd_state[i] & CEPH_OSD_BACKFILLFULL) {
+        if (get_osd_utilization(osd_stat, i, &kb, &kb_used, &kb_avail))
+         backfill->emplace(i, (float)kb_used / (float)kb);
+      } else if (osd_state[i] & CEPH_OSD_NEARFULL) {
+        if (get_osd_utilization(osd_stat, i, &kb, &kb_used, &kb_avail))
+         nearfull->emplace(i, (float)kb_used / (float)kb);
+      }
+    }
+  }
+}
+
 void OSDMap::get_all_osds(set<int32_t>& ls) const
 {
   for (int i=0; i<max_osd; i++)

diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 2e8fcf800d9879752e44c4c1dc3c6133515aec4b..525ba12fb0e690a59d9f5ea0ab9811bd83a24647 100644 (file)
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -344,6 +344,8 @@ public:
     return nearfull_ratio;
   }
   void count_full_nearfull_osds(int *full, int *backfill, int *nearfull) const;
+  void get_full_osd_util(const ceph::unordered_map<int32_t,osd_stat_t> &osd_stat,
+                         map<int, float> *full, map<int, float> *backfill, map<int, float> *nearfull) const;
 
   /***** cluster state *****/
   /* osds */