From: Samuel Just Date: Tue, 21 Feb 2017 05:24:33 +0000 (-0800) Subject: PrimaryLogPG: reimplement osd_snap_trim_sleep within the state machine X-Git-Tag: v12.0.1~273^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2ed7759cfeb03e71f0fbd98fe7c2db2bb741861c;p=ceph.git PrimaryLogPG: reimplement osd_snap_trim_sleep within the state machine Rather than blocking the main op queue, just pause for that amount of time between state machine cycles. Also, add osd_snap_trim_sleep to a few of the thrasher yamls. Signed-off-by: Samuel Just --- diff --git a/qa/suites/rados/thrash/thrashers/default.yaml b/qa/suites/rados/thrash/thrashers/default.yaml index f5e432d3fca0..5df84bedabeb 100644 --- a/qa/suites/rados/thrash/thrashers/default.yaml +++ b/qa/suites/rados/thrash/thrashers/default.yaml @@ -10,6 +10,7 @@ tasks: osd scrub min interval: 60 osd scrub max interval: 120 osd max backfills: 3 + osd snap trim sleep: 2 - thrashosds: timeout: 1200 chance_pgnum_grow: 1 diff --git a/qa/suites/rados/thrash/thrashers/pggrow.yaml b/qa/suites/rados/thrash/thrashers/pggrow.yaml index d381026af3b6..30d8957c77b4 100644 --- a/qa/suites/rados/thrash/thrashers/pggrow.yaml +++ b/qa/suites/rados/thrash/thrashers/pggrow.yaml @@ -10,6 +10,7 @@ tasks: osd scrub max interval: 120 filestore odsync write: true osd max backfills: 2 + osd snap trim sleep: .5 - thrashosds: timeout: 1200 chance_pgnum_grow: 2 diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 16e9804bcf2e..6568f36ca74b 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -757,7 +757,7 @@ OPTION(osd_op_thread_suicide_timeout, OPT_INT, 150) OPTION(osd_recovery_thread_timeout, OPT_INT, 30) OPTION(osd_recovery_thread_suicide_timeout, OPT_INT, 300) OPTION(osd_recovery_sleep, OPT_FLOAT, 0) // seconds to sleep between recovery ops -OPTION(osd_snap_trim_sleep, OPT_FLOAT, 0) +OPTION(osd_snap_trim_sleep, OPT_DOUBLE, 0) OPTION(osd_scrub_invalid_stats, OPT_BOOL, true) OPTION(osd_remove_thread_timeout, OPT_INT, 60*60) OPTION(osd_remove_thread_suicide_timeout, OPT_INT, 10*60*60) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 1d77f5aa6d40..533af907605a 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -261,6 +261,9 @@ OSDService::OSDService(OSD *osd) : remote_reserver(&reserver_finisher, cct->_conf->osd_max_backfills, cct->_conf->osd_min_recovery_priority), pg_temp_lock("OSDService::pg_temp_lock"), + snap_sleep_lock("OSDService::snap_sleep_lock"), + snap_sleep_timer( + osd->client_messenger->cct, snap_sleep_lock, false /* relax locking */), snap_reserver(&reserver_finisher, cct->_conf->osd_max_trimming_pgs), recovery_lock("OSDService::recovery_lock"), @@ -493,6 +496,12 @@ void OSDService::shutdown() Mutex::Locker l(backfill_request_lock); backfill_request_timer.shutdown(); } + + { + Mutex::Locker l(snap_sleep_lock); + snap_sleep_timer.shutdown(); + } + osdmap = OSDMapRef(); next_osdmap = OSDMapRef(); } @@ -504,6 +513,7 @@ void OSDService::init() objecter->set_client_incarnation(0); watch_timer.init(); agent_timer.init(); + snap_sleep_timer.init(); agent_thread.create("osd_srv_agent"); diff --git a/src/osd/OSD.h b/src/osd/OSD.h index c43be7a3662e..f0ff927ab108 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -912,6 +912,9 @@ public: void queue_for_peering(PG *pg); + Mutex snap_sleep_lock; + SafeTimer snap_sleep_timer; + AsyncReserver snap_reserver; void queue_for_snap_trim(PG *pg); diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc index 70e3b48c9683..8dc67cb6667d 100644 --- a/src/osd/PrimaryLogPG.cc +++ b/src/osd/PrimaryLogPG.cc @@ -3734,14 +3734,6 @@ void PrimaryLogPG::snap_trimmer(epoch_t queued) if (deleting || pg_has_reset_since(queued)) { return; } - if (cct->_conf->osd_snap_trim_sleep > 0) { - unlock(); - utime_t t; - t.set_from_double(cct->_conf->osd_snap_trim_sleep); - t.sleep(); - lock(); - dout(20) << __func__ << " slept for " << t << dendl; - } assert(is_primary()); diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h index 38325c8c4033..20128ebd5c14 100644 --- a/src/osd/PrimaryLogPG.h +++ b/src/osd/PrimaryLogPG.h @@ -1451,6 +1451,9 @@ private: struct SnapTrimReserved : boost::statechart::event< SnapTrimReserved > { SnapTrimReserved() : boost::statechart::event< SnapTrimReserved >() {} }; + struct SnapTrimTimerReady : boost::statechart::event< SnapTrimTimerReady > { + SnapTrimTimerReady() : boost::statechart::event< SnapTrimTimerReady >() {} + }; struct NotTrimming; struct SnapTrimmer : public boost::statechart::state_machine< SnapTrimmer, NotTrimming > { @@ -1493,6 +1496,57 @@ private: }; /* SnapTrimmerStates */ + struct WaitTrimTimer : boost::statechart::state< WaitTrimTimer, Trimming >, NamedState { + typedef boost::mpl::list < + boost::statechart::custom_reaction< SnapTrimTimerReady > + > reactions; + Context *wakeup = nullptr; + explicit WaitTrimTimer(my_context ctx) + : my_base(ctx), + NamedState(context< SnapTrimmer >().pg->cct, "Trimming/WaitTrimTimer") { + context< SnapTrimmer >().log_enter(state_name); + assert(context().in_flight.empty()); + struct OnTimer : Context { + PrimaryLogPGRef pg; + epoch_t epoch; + OnTimer(PrimaryLogPGRef pg, epoch_t epoch) : pg(pg), epoch(epoch) {} + void finish(int) override { + pg->lock(); + if (!pg->pg_has_reset_since(epoch)) + pg->snap_trimmer_machine.process_event(SnapTrimTimerReady()); + pg->unlock(); + } + }; + auto *pg = context< SnapTrimmer >().pg; + if (pg->cct->_conf->osd_snap_trim_sleep > 0) { + wakeup = new OnTimer{pg, pg->get_osdmap()->get_epoch()}; + Mutex::Locker l(pg->osd->snap_sleep_lock); + pg->osd->snap_sleep_timer.add_event_after( + pg->cct->_conf->osd_snap_trim_sleep, wakeup); + } else { + post_event(SnapTrimTimerReady()); + } + } + void exit() { + context< SnapTrimmer >().log_exit(state_name, enter_time); + auto *pg = context< SnapTrimmer >().pg; + if (wakeup) { + Mutex::Locker l(pg->osd->snap_sleep_lock); + pg->osd->snap_sleep_timer.cancel_event(wakeup); + wakeup = nullptr; + } + } + boost::statechart::result react(const SnapTrimTimerReady &) { + wakeup = nullptr; + if (!context< SnapTrimmer >().can_trim()) { + post_event(KickTrim()); + return transit< NotTrimming >(); + } else { + return transit< AwaitAsyncWork >(); + } + } + }; + struct WaitRWLock : boost::statechart::state< WaitRWLock, Trimming >, NamedState { typedef boost::mpl::list < boost::statechart::custom_reaction< TrimWriteUnblocked > @@ -1534,7 +1588,7 @@ private: post_event(KickTrim()); return transit< NotTrimming >(); } else { - return transit< AwaitAsyncWork >(); + return transit< WaitTrimTimer >(); } } };