ss << tab;
}
-
-
-// Only called with a single bit set in "what"
-static void note_stuck_detail(
- int what,
- mempool::pgmap::unordered_map<pg_t,pg_stat_t>& stuck_pgs,
- int max_detail,
- list<pair<health_status_t,string> > *detail)
-{
- int n = 0;
- for (auto p = stuck_pgs.begin();
- p != stuck_pgs.end();
- ++p) {
- ostringstream ss;
- utime_t since;
- const char *whatname = 0;
- switch (what) {
- case PGMap::STUCK_INACTIVE:
- since = p->second.last_active;
- whatname = "inactive";
- break;
- case PGMap::STUCK_UNCLEAN:
- since = p->second.last_clean;
- whatname = "unclean";
- break;
- case PGMap::STUCK_DEGRADED:
- since = p->second.last_undegraded;
- whatname = "degraded";
- break;
- case PGMap::STUCK_UNDERSIZED:
- since = p->second.last_fullsized;
- whatname = "undersized";
- break;
- case PGMap::STUCK_STALE:
- since = p->second.last_unstale;
- whatname = "stale";
- break;
- default:
- ceph_abort();
- }
- if (--max_detail == 0) {
- ostringstream ss;
- ss << (stuck_pgs.size() - n) << " more pgs are also stuck " << whatname;
- detail->push_back(make_pair(HEALTH_WARN, ss.str()));
- break;
- }
- ++n;
- ss << "pg " << p->first << " is stuck " << whatname;
- if (since == utime_t()) {
- ss << " since forever";
- } else {
- utime_t dur = ceph_clock_now() - since;
- ss << " for " << dur;
- }
- ss << ", current state " << pg_state_string(p->second.state)
- << ", last acting " << p->second.acting;
- detail->push_back(make_pair(HEALTH_WARN, ss.str()));
- }
-}
-
-static pair<int,int> _warn_slow_request_histogram(
- CephContext *cct,
- const pow2_hist_t& h,
- string suffix,
- list<pair<health_status_t,string> >& summary,
- list<pair<health_status_t,string> > *detail)
-{
- if (h.h.empty())
- return make_pair(0, 0);
-
- unsigned warn = 0, error = 0;
- float err_age =
- cct->_conf->mon_osd_warn_op_age * cct->_conf->mon_osd_err_op_age_ratio;
- for (unsigned i = h.h.size() - 1; i > 0; --i) {
- float ub = (float)(1 << i) / 1000.0;
- if (ub < cct->_conf->mon_osd_warn_op_age)
- break;
- if (h.h[i]) {
- auto sev = HEALTH_WARN;
- if (ub > err_age) {
- sev = HEALTH_ERR;
- error += h.h[i];
- } else {
- warn += h.h[i];
- }
- if (detail) {
- ostringstream ss;
- ss << h.h[i] << " ops are blocked > " << ub << " sec" << suffix;
- detail->push_back(make_pair(sev, ss.str()));
- }
- }
- }
- return make_pair(warn, error);
-}
-
-namespace {
- enum class scrubbed_or_deepscrubbed_t { SCRUBBED, DEEPSCRUBBED };
-
- void print_unscrubbed_detailed(
- const std::pair<const pg_t,pg_stat_t> &pg_entry,
- list<pair<health_status_t,string> > *detail,
- scrubbed_or_deepscrubbed_t how_scrubbed)
- {
- std::stringstream ss;
- const auto& pg_stat(pg_entry.second);
-
- ss << "pg " << pg_entry.first << " is not ";
- if (how_scrubbed == scrubbed_or_deepscrubbed_t::SCRUBBED) {
- ss << "scrubbed, last_scrub_stamp "
- << pg_stat.last_scrub_stamp;
- } else if (how_scrubbed == scrubbed_or_deepscrubbed_t::DEEPSCRUBBED) {
- ss << "deep-scrubbed, last_deep_scrub_stamp "
- << pg_stat.last_deep_scrub_stamp;
- }
-
- detail->push_back(make_pair(HEALTH_WARN, ss.str()));
- }
-
- using pg_stat_map_t = const mempool::pgmap::unordered_map<pg_t,pg_stat_t>;
-
- void print_unscrubbed_pgs(
- pg_stat_map_t& pg_stats,
- list<pair<health_status_t,string> > &summary,
- list<pair<health_status_t,string> > *detail,
- const CephContext* cct)
- {
- if (cct->_conf->mon_warn_not_scrubbed == 0 &&
- cct->_conf->mon_warn_not_deep_scrubbed == 0)
- return;
-
- int pgs_count = 0;
- const utime_t now = ceph_clock_now();
- for (const auto& pg_entry : pg_stats) {
- const auto& pg_stat(pg_entry.second);
- const utime_t time_since_ls = now - pg_stat.last_scrub_stamp;
- const utime_t time_since_lds = now - pg_stat.last_deep_scrub_stamp;
-
- const int mon_warn_not_scrubbed =
- cct->_conf->mon_warn_not_scrubbed + cct->_conf->mon_scrub_interval;
-
- const int mon_warn_not_deep_scrubbed =
- cct->_conf->mon_warn_not_deep_scrubbed + cct->_conf->osd_deep_scrub_interval;
-
- bool not_scrubbed = (time_since_ls >= mon_warn_not_scrubbed &&
- cct->_conf->mon_warn_not_scrubbed != 0);
-
- bool not_deep_scrubbed = (time_since_lds >= mon_warn_not_deep_scrubbed &&
- cct->_conf->mon_warn_not_deep_scrubbed != 0);
-
- if (detail != nullptr) {
- if (not_scrubbed) {
- print_unscrubbed_detailed(pg_entry,
- detail,
- scrubbed_or_deepscrubbed_t::SCRUBBED);
- }
- if (not_deep_scrubbed) {
- print_unscrubbed_detailed(pg_entry,
- detail,
- scrubbed_or_deepscrubbed_t::DEEPSCRUBBED);
- }
- }
- if (not_scrubbed || not_deep_scrubbed) {
- ++pgs_count;
- }
- }
-
- if (pgs_count > 0) {
- std::stringstream ss;
- ss << pgs_count << " unscrubbed pgs";
- summary.push_back(make_pair(HEALTH_WARN, ss.str()));
- }
- }
-}
-
void PGMap::get_health_checks(
CephContext *cct,
const OSDMap& osdmap,