PrimaryLogPG: reimplement osd_snap_trim_sleep within the state machine

Rather than blocking the main op queue, just pause for that amount of
time between state machine cycles.

Also, add osd_snap_trim_sleep to a few of the thrasher yamls.

Signed-off-by: Samuel Just <sjust@redhat.com>

diff --git a/qa/suites/rados/thrash/thrashers/default.yaml b/qa/suites/rados/thrash/thrashers/default.yaml
index f5e432d3fca050413d63a5bc67df887cf964a068..5df84bedabeb1768632e78891e7e8fdf0800cc3c 100644 (file)
--- a/qa/suites/rados/thrash/thrashers/default.yaml
+++ b/qa/suites/rados/thrash/thrashers/default.yaml
@@ -10,6 +10,7 @@ tasks:
         osd scrub min interval: 60
         osd scrub max interval: 120
         osd max backfills: 3
+        osd snap trim sleep: 2
 - thrashosds:
     timeout: 1200
     chance_pgnum_grow: 1

diff --git a/qa/suites/rados/thrash/thrashers/pggrow.yaml b/qa/suites/rados/thrash/thrashers/pggrow.yaml
index d381026af3b6f74b53d4d5987f09c6f4218bb398..30d8957c77b4a91bf3d00db65e75c8daa63e185f 100644 (file)
--- a/qa/suites/rados/thrash/thrashers/pggrow.yaml
+++ b/qa/suites/rados/thrash/thrashers/pggrow.yaml
@@ -10,6 +10,7 @@ tasks:
         osd scrub max interval: 120
         filestore odsync write: true
         osd max backfills: 2
+        osd snap trim sleep: .5
 - thrashosds:
     timeout: 1200
     chance_pgnum_grow: 2

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 16e9804bcf2e7c37ea19f42a5e6e43fba2bdfb66..6568f36ca74b7f964321b090ade0967c8a73f852 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -757,7 +757,7 @@ OPTION(osd_op_thread_suicide_timeout, OPT_INT, 150)
 OPTION(osd_recovery_thread_timeout, OPT_INT, 30)
 OPTION(osd_recovery_thread_suicide_timeout, OPT_INT, 300)
 OPTION(osd_recovery_sleep, OPT_FLOAT, 0)         // seconds to sleep between recovery ops
-OPTION(osd_snap_trim_sleep, OPT_FLOAT, 0)
+OPTION(osd_snap_trim_sleep, OPT_DOUBLE, 0)
 OPTION(osd_scrub_invalid_stats, OPT_BOOL, true)
 OPTION(osd_remove_thread_timeout, OPT_INT, 60*60)
 OPTION(osd_remove_thread_suicide_timeout, OPT_INT, 10*60*60)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 1d77f5aa6d40588ba910a9010a4c396e936b92d9..533af907605a4426f3f35cadafb922a77145c436 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -261,6 +261,9 @@ OSDService::OSDService(OSD *osd) :
   remote_reserver(&reserver_finisher, cct->_conf->osd_max_backfills,
                  cct->_conf->osd_min_recovery_priority),
   pg_temp_lock("OSDService::pg_temp_lock"),
+  snap_sleep_lock("OSDService::snap_sleep_lock"),
+  snap_sleep_timer(
+    osd->client_messenger->cct, snap_sleep_lock, false /* relax locking */),
   snap_reserver(&reserver_finisher,
                cct->_conf->osd_max_trimming_pgs),
   recovery_lock("OSDService::recovery_lock"),
@@ -493,6 +496,12 @@ void OSDService::shutdown()
     Mutex::Locker l(backfill_request_lock);
     backfill_request_timer.shutdown();
   }
+
+  {
+    Mutex::Locker l(snap_sleep_lock);
+    snap_sleep_timer.shutdown();
+  }
+
   osdmap = OSDMapRef();
   next_osdmap = OSDMapRef();
 }
@@ -504,6 +513,7 @@ void OSDService::init()
   objecter->set_client_incarnation(0);
   watch_timer.init();
   agent_timer.init();
+  snap_sleep_timer.init();
 
   agent_thread.create("osd_srv_agent");
 

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index c43be7a3662ecd97fd4df38c82811faa7056bcdc..f0ff927ab10835c7a6b0a84c883f24ca03b45467 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -912,6 +912,9 @@ public:
 
   void queue_for_peering(PG *pg);
 
+  Mutex snap_sleep_lock;
+  SafeTimer snap_sleep_timer;
+
   AsyncReserver<spg_t> snap_reserver;
   void queue_for_snap_trim(PG *pg);
 

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 70e3b48c9683dbe616ef632302bd17de08f73b81..8dc67cb6667d8104d0fa247bc7f41f533dc12439 100644 (file)
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -3734,14 +3734,6 @@ void PrimaryLogPG::snap_trimmer(epoch_t queued)
   if (deleting || pg_has_reset_since(queued)) {
     return;
   }
-  if (cct->_conf->osd_snap_trim_sleep > 0) {
-    unlock();
-    utime_t t;
-    t.set_from_double(cct->_conf->osd_snap_trim_sleep);
-    t.sleep();
-    lock();
-    dout(20) << __func__ << " slept for " << t << dendl;
-  }
 
   assert(is_primary());
 

diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h
index 38325c8c4033435a5400839214234e895a33176d..20128ebd5c1430552d504b7297784f3a8579a9ea 100644 (file)
--- a/src/osd/PrimaryLogPG.h
+++ b/src/osd/PrimaryLogPG.h
@@ -1451,6 +1451,9 @@ private:
   struct SnapTrimReserved : boost::statechart::event< SnapTrimReserved > {
     SnapTrimReserved() : boost::statechart::event< SnapTrimReserved >() {}
   };
+  struct SnapTrimTimerReady : boost::statechart::event< SnapTrimTimerReady > {
+    SnapTrimTimerReady() : boost::statechart::event< SnapTrimTimerReady >() {}
+  };
 
   struct NotTrimming;
   struct SnapTrimmer : public boost::statechart::state_machine< SnapTrimmer, NotTrimming > {
@@ -1493,6 +1496,57 @@ private:
   };
 
   /* SnapTrimmerStates */
+  struct WaitTrimTimer : boost::statechart::state< WaitTrimTimer, Trimming >, NamedState {
+    typedef boost::mpl::list <
+      boost::statechart::custom_reaction< SnapTrimTimerReady >
+      > reactions;
+    Context *wakeup = nullptr;
+    explicit WaitTrimTimer(my_context ctx)
+      : my_base(ctx),
+       NamedState(context< SnapTrimmer >().pg->cct, "Trimming/WaitTrimTimer") {
+      context< SnapTrimmer >().log_enter(state_name);
+      assert(context<Trimming>().in_flight.empty());
+      struct OnTimer : Context {
+       PrimaryLogPGRef pg;
+       epoch_t epoch;
+       OnTimer(PrimaryLogPGRef pg, epoch_t epoch) : pg(pg), epoch(epoch) {}
+       void finish(int) override {
+         pg->lock();
+         if (!pg->pg_has_reset_since(epoch))
+           pg->snap_trimmer_machine.process_event(SnapTrimTimerReady());
+         pg->unlock();
+       }
+      };
+      auto *pg = context< SnapTrimmer >().pg;
+      if (pg->cct->_conf->osd_snap_trim_sleep > 0) {
+       wakeup = new OnTimer{pg, pg->get_osdmap()->get_epoch()};
+       Mutex::Locker l(pg->osd->snap_sleep_lock);
+       pg->osd->snap_sleep_timer.add_event_after(
+         pg->cct->_conf->osd_snap_trim_sleep, wakeup);
+      } else {
+       post_event(SnapTrimTimerReady());
+      }
+    }
+    void exit() {
+      context< SnapTrimmer >().log_exit(state_name, enter_time);
+      auto *pg = context< SnapTrimmer >().pg;
+      if (wakeup) {
+       Mutex::Locker l(pg->osd->snap_sleep_lock);
+       pg->osd->snap_sleep_timer.cancel_event(wakeup);
+       wakeup = nullptr;
+      }
+    }
+    boost::statechart::result react(const SnapTrimTimerReady &) {
+      wakeup = nullptr;
+      if (!context< SnapTrimmer >().can_trim()) {
+       post_event(KickTrim());
+       return transit< NotTrimming >();
+      } else {
+       return transit< AwaitAsyncWork >();
+      }
+    }
+  };
+
   struct WaitRWLock : boost::statechart::state< WaitRWLock, Trimming >, NamedState {
     typedef boost::mpl::list <
       boost::statechart::custom_reaction< TrimWriteUnblocked >
@@ -1534,7 +1588,7 @@ private:
        post_event(KickTrim());
        return transit< NotTrimming >();
       } else {
-       return transit< AwaitAsyncWork >();
+       return transit< WaitTrimTimer >();
       }
     }
   };