osd/PG: bound number of blocked_by OSDs we report per PG

We don't want to include an unbounded-sized vector of OSDs in the pg_stat_t
struct.  Cap it, and report a random subset of the actual blockers to fit
within the limit.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index b07b6a7ed24652af7ac4497af16893e3956c002d..f8b263c19b3d3a12973caad18f452b8942b12187 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -538,6 +538,7 @@ OPTION(osd_min_pg_log_entries, OPT_U32, 3000)  // number of entries to keep in t
 OPTION(osd_max_pg_log_entries, OPT_U32, 10000) // max entries, say when degraded, before we trim
 OPTION(osd_op_complaint_time, OPT_FLOAT, 30) // how many seconds old makes an op complaint-worthy
 OPTION(osd_command_max_records, OPT_INT, 256)
+OPTION(osd_max_pg_blocked_by, OPT_U32, 16)    // max peer osds to report that are blocking our progress
 OPTION(osd_op_log_threshold, OPT_INT, 5) // how many op log messages to show in one go
 OPTION(osd_verify_sparse_read_holes, OPT_BOOL, false)  // read fiemap-reported holes and verify they are zeros
 OPTION(osd_debug_drop_ping_probability, OPT_DOUBLE, 0)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index df51ef4c5487ee24e7b6b87223807e08ea148085..52a146907d4f6d767e48c1ad298e014d5f5d10aa 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -2245,6 +2245,27 @@ void PG::_update_calc_stats()
   }
 }
 
+void PG::_update_blocked_by()
+{
+  // set a max on the number of blocking peers we report. if we go
+  // over, report a random subset.  keep the result sorted.
+  unsigned keep = MIN(blocked_by.size(), g_conf->osd_max_pg_blocked_by);
+  unsigned skip = blocked_by.size() - keep;
+  info.stats.blocked_by.clear();
+  info.stats.blocked_by.resize(keep);
+  unsigned pos = 0;
+  for (set<int>::iterator p = blocked_by.begin();
+       p != blocked_by.end() && keep > 0;
+       ++p) {
+    if (skip > 0 && (rand() % (skip + keep) < skip)) {
+      --skip;
+    } else {
+      info.stats.blocked_by[pos++] = *p;
+      --keep;
+    }
+  }
+}
+
 void PG::publish_stats_to_osd()
 {
   pg_stats_publish_lock.Lock();
@@ -2274,12 +2295,7 @@ void PG::publish_stats_to_osd()
     info.stats.last_unstale = now;
 
     _update_calc_stats();
-
-    info.stats.blocked_by.clear();
-    info.stats.blocked_by.resize(blocked_by.size());
-    unsigned pos = 0;
-    for (set<int>::iterator p = blocked_by.begin(); p != blocked_by.end(); ++p)
-      info.stats.blocked_by[pos++] = *p;
+    _update_blocked_by();
 
     pg_stats_publish_valid = true;
     pg_stats_publish = info.stats;

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 714b1fc679784228bb9ebdbb1a3eca99943ecc12..02b575856658a5e5fa4d22795dd9223b8fd892c8 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -751,6 +751,7 @@ protected:
   ceph::shared_ptr<ObjectStore::Sequencer> osr;
 
   void _update_calc_stats();
+  void _update_blocked_by();
   void publish_stats_to_osd();
   void clear_publish_stats();