rgw: LatencyConcurrencyControl warns on very high latency

Limited to only warn every five minutes.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>

diff --git a/src/rgw/driver/rados/rgw_data_sync.h b/src/rgw/driver/rados/rgw_data_sync.h
index 9d1b0e77f8e1e507620cfa821687f29d8c19924a..ccaa20884eb561a8328fef36553a6c5ecc01bcc8 100644 (file)
--- a/src/rgw/driver/rados/rgw_data_sync.h
+++ b/src/rgw/driver/rados/rgw_data_sync.h
@@ -356,6 +356,8 @@ void pretty_print(const RGWDataSyncEnv* env, const S& fmt, T&& ...t) {
 /// Keep a running average of operation latency and scale concurrency
 /// down when latency rises.
 class LatencyConcurrencyControl : public LatencyMonitor {
+  static constexpr auto dout_subsys = ceph_subsys_rgw;
+  ceph::coarse_mono_time last_warning;
 public:
   CephContext* cct;
 
@@ -373,6 +375,14 @@ public:
     auto threshold = (cct->_conf->rgw_sync_lease_period * 1s) / 12;
 
     if (avg_latency() >= 2 * threshold) [[unlikely]] {
+      auto now = ceph::coarse_mono_clock::now();
+      if (now - last_warning > 5min) {
+        ldout(cct, -1)
+            << "WARNING: The OSD cluster is overloaded and struggling to "
+            << "complete ops. You need more capacity to serve this level "
+           << "of demand." << dendl;
+       last_warning = now;
+      }
       return 1;
     } else if (avg_latency() >= threshold) [[unlikely]] {
       return concurrency / 2;