osd: Implement asynchronous recovery sleep

Signed-off-by: Neha Ojha <nojha@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index e6d18ecd2df6702ee997928f8df63ad72a133df5..b2eaa39eac3b0e9f8adcfd31d884a474ca9e857f 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -256,6 +256,8 @@ OSDService::OSDService(OSD *osd) :
   next_notif_id(0),
   recovery_request_lock("OSDService::recovery_request_lock"),
   recovery_request_timer(cct, recovery_request_lock, false),
+  recovery_sleep_lock("OSDService::recovery_sleep_lock"),
+  recovery_sleep_timer(cct, recovery_sleep_lock, false),
   reserver_finisher(cct),
   local_reserver(&reserver_finisher, cct->_conf->osd_max_backfills,
                 cct->_conf->osd_min_recovery_priority),
@@ -498,6 +500,11 @@ void OSDService::shutdown()
     recovery_request_timer.shutdown();
   }
 
+  {
+    Mutex::Locker l(recovery_sleep_lock);
+    recovery_sleep_timer.shutdown();
+  }
+
   {
     Mutex::Locker l(snap_sleep_lock);
     snap_sleep_timer.shutdown();
@@ -2225,6 +2232,7 @@ int OSD::init()
   tick_timer.init();
   tick_timer_without_osd_lock.init();
   service.recovery_request_timer.init();
+  service.recovery_sleep_timer.init();
 
   // mount.
   dout(2) << "mounting " << dev_path << " "
@@ -8881,18 +8889,35 @@ void OSD::do_recovery(
   ThreadPool::TPHandle &handle)
 {
   uint64_t started = 0;
-  if (cct->_conf->osd_recovery_sleep > 0) {
-    handle.suspend_tp_timeout();
-    pg->unlock();
-    utime_t t;
-    t.set_from_double(cct->_conf->osd_recovery_sleep);
-    t.sleep();
-    dout(20) << __func__ << " slept for " << t << dendl;
-    pg->lock();
-    handle.reset_tp_timeout();
+  if (cct->_conf->osd_recovery_sleep > 0 && service.recovery_needs_sleep) {
+    auto recovery_requeue_callback = new FunctionContext([this, pg](int r) {
+      pg->lock();
+      dout(20) << "do_recovery wake up at "
+               << ceph_clock_now()
+              << ", re-queuing recovery" << dendl;
+      service.recovery_needs_sleep = false;
+      pg->recovery_queued = false;
+      pg->queue_recovery();
+      pg->unlock();
+    });
+    Mutex::Locker l(service.recovery_sleep_lock);
+
+    // This is true for the first recovery op and when the previous recovery op
+    // has been scheduled in the past. The next recovery op is scheduled after
+    // completing the sleep from now.
+    if (service.recovery_schedule_time < ceph_clock_now()) {
+      service.recovery_schedule_time = ceph_clock_now();
+    }
+    service.recovery_schedule_time += cct->_conf->osd_recovery_sleep;
+    service.recovery_sleep_timer.add_event_at(service.recovery_schedule_time,
+                                             recovery_requeue_callback);
+    dout(20) << "Recovery event scheduled at "
+             << service.recovery_schedule_time << dendl;
+    return;
   }
 
   {
+    service.recovery_needs_sleep = true;
     if (pg->pg_has_reset_since(queued)) {
       goto out;
     }

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index cad51ed9264357feab9b6804030baa0ea594bb21..504d9c1dec660f16f3f73c0ed84e848e1c519b27 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -923,6 +923,13 @@ public:
   Mutex recovery_request_lock;
   SafeTimer recovery_request_timer;
 
+  // For async recovery sleep
+  bool recovery_needs_sleep = true;
+  utime_t recovery_schedule_time = utime_t();
+
+  Mutex recovery_sleep_lock;
+  SafeTimer recovery_sleep_timer;
+
   // -- tids --
   // for ops i issue
   std::atomic_uint last_tid{0};