osd: re-queue recovery without pg lock

Signed-off-by: Neha Ojha <nojha@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index b2eaa39eac3b0e9f8adcfd31d884a474ca9e857f..5fe26ba86ad3678951df1b21ea48d7fe49e2e938 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -480,6 +480,11 @@ void OSDService::start_shutdown()
     Mutex::Locker l(agent_timer_lock);
     agent_timer.shutdown();
   }
+
+  {
+    Mutex::Locker l(recovery_sleep_lock);
+    recovery_sleep_timer.shutdown();
+  }
 }
 
 void OSDService::shutdown()
@@ -500,11 +505,6 @@ void OSDService::shutdown()
     recovery_request_timer.shutdown();
   }
 
-  {
-    Mutex::Locker l(recovery_sleep_lock);
-    recovery_sleep_timer.shutdown();
-  }
-
   {
     Mutex::Locker l(snap_sleep_lock);
     snap_sleep_timer.shutdown();
@@ -8889,16 +8889,21 @@ void OSD::do_recovery(
   ThreadPool::TPHandle &handle)
 {
   uint64_t started = 0;
+
+  /*
+   * When the value of osd_recovery_sleep is set greater than zero, recovery
+   * ops are scheduled after osd_recovery_sleep amount of time from the previous
+   * recovery event's schedule time. This is done by adding a
+   * recovery_requeue_callback event, which re-queues the recovery op using
+   * queue_recovery_after_sleep.
+   */
   if (cct->_conf->osd_recovery_sleep > 0 && service.recovery_needs_sleep) {
-    auto recovery_requeue_callback = new FunctionContext([this, pg](int r) {
-      pg->lock();
+    auto recovery_requeue_callback = new FunctionContext([this, pg, queued, reserved_pushes](int r) {
       dout(20) << "do_recovery wake up at "
                << ceph_clock_now()
               << ", re-queuing recovery" << dendl;
       service.recovery_needs_sleep = false;
-      pg->recovery_queued = false;
-      pg->queue_recovery();
-      pg->unlock();
+      service.queue_recovery_after_sleep(pg, queued, reserved_pushes);
     });
     Mutex::Locker l(service.recovery_sleep_lock);
 

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 504d9c1dec660f16f3f73c0ed84e848e1c519b27..7ce8cbaf63d25cf96e6281d7969b91166995c1a1 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1057,6 +1057,10 @@ public:
     }
     _maybe_queue_recovery();
   }
+  void queue_recovery_after_sleep(PG *pg, epoch_t queued, uint64_t reserved_pushes) {
+    Mutex::Locker l(recovery_lock);
+    _queue_for_recovery(make_pair(queued, pg), reserved_pushes);
+  }
 
 
   // osd map cache (past osd maps)