From 69eea16f39f2255962041de29d6393ecc37bb0cb Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 19 Nov 2008 11:14:53 -0800 Subject: [PATCH] osd: convert snap trimming to snap_trim_wq --- src/common/WorkQueue.h | 2 +- src/osd/OSD.cc | 50 ++++++++++-------------------------------- src/osd/OSD.h | 42 +++++++++++++++++++++-------------- src/osd/PG.cc | 6 +---- src/osd/PG.h | 7 ++---- 5 files changed, 41 insertions(+), 66 deletions(-) diff --git a/src/common/WorkQueue.h b/src/common/WorkQueue.h index 4f1c590a782fd..ec2e193452bf3 100644 --- a/src/common/WorkQueue.h +++ b/src/common/WorkQueue.h @@ -93,7 +93,7 @@ public: cond.Signal(); queue_lock.Unlock(); } - void unqueue(T *item) { + void dequeue(T *item) { queue_lock.Lock(); _dequeue(item); queue_lock.Unlock(); diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 74d812abb2d37..acccd4bd75c42 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -273,14 +273,13 @@ OSD::OSD(int id, Messenger *m, MonMap *mm, const char *dev) : osdmap(NULL), map_lock("OSD::map_lock"), map_cache_lock("OSD::map_cache_lock"), - snap_trimmer_lock("OSD::snap_trimmer_lock"), - snap_trimmer_thread(this), pg_stat_queue_lock("OSD::pg_stat_queue_lock"), tid_lock("OSD::tid_lock"), recovery_lock("OSD::recovery_lock"), recovery_ops_active(0), recovery_stop(false), recovery_pause(false), remove_list_lock("OSD::remove_list_lock"), recovery_thread(this), + snap_trim_wq(this), scrub_wq(this) { messenger = m; @@ -440,6 +439,7 @@ int OSD::init() recovery_thread.create(); scrub_wq.start(); + snap_trim_wq.start(); // start the heartbeat timer.add_event_after(g_conf.osd_heartbeat_interval, new C_Heartbeat(this)); @@ -478,7 +478,10 @@ int OSD::shutdown() dout(10) << "recovery thread stopped" << dendl; scrub_wq.stop(); - dout(10) << "scrub thread stopped" << dendl; + dout(10) << "scrub wq stopped" << dendl; + snap_trim_wq.stop(); + dout(10) << "snap trim wq stopped" << dendl; + // zap waiters (bleh, this is messy) finished_lock.Lock(); @@ -658,6 +661,9 @@ void OSD::_remove_unlock_pg(PG *pg) dout(10) << "_remove_unlock_pg " << pgid << dendl; + snap_trim_wq.dequeue(pg); + scrub_wq.dequeue(pg); + // remove from store vector olist; @@ -1509,6 +1515,7 @@ void OSD::handle_osd_map(MOSDMap *m) wait_for_no_ops(); pause_recovery_thread(); scrub_wq.pause(); + snap_trim_wq.pause(); map_lock.get_write(); assert(osd_lock.is_locked()); @@ -1703,6 +1710,7 @@ void OSD::handle_osd_map(MOSDMap *m) map_lock.put_write(); unpause_recovery_thread(); scrub_wq.unpause(); + snap_trim_wq.unpause(); //if (osdmap->get_epoch() == 1) store->sync(); // in case of early death, blah @@ -3391,39 +3399,3 @@ void OSD::wait_for_no_ops() - -void OSD::wake_snap_trimmer() -{ - assert(osd_lock.is_locked()); - if (!snap_trimmer_thread.is_started()) { - dout(10) << "wake_snap_trimmer - creating thread" << dendl; - snap_trimmer_thread.create(); - } else { - dout(10) << "wake_snap_trimmer - kicking thread" << dendl; - snap_trimmer_cond.Signal(); - } -} - -void OSD::snap_trimmer() -{ - osd_lock.Lock(); - while (1) { - snap_trimmer_lock.Lock(); - if (pgs_pending_snap_removal.empty()) { - snap_trimmer_lock.Unlock(); - dout(10) << "snap_trimmer - no pgs pending trim, sleeping" << dendl; - snap_trimmer_cond.Wait(osd_lock); - continue; - } - - PG *pg = pgs_pending_snap_removal.front(); - pgs_pending_snap_removal.pop_front(); - snap_trimmer_lock.Unlock(); - osd_lock.Unlock(); - - pg->snap_trimmer(); - - osd_lock.Lock(); - } - osd_lock.Unlock(); -} diff --git a/src/osd/OSD.h b/src/osd/OSD.h index 0442480e64554..d23ddde5c719f 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -284,7 +284,6 @@ private: // -- placement groups -- hash_map pg_map; hash_map > waiting_for_pg; - xlist pgs_pending_snap_removal; bool _have_pg(pg_t pgid); PG *_lookup_lock_pg(pg_t pgid); @@ -298,21 +297,6 @@ private: void project_pg_history(pg_t pgid, PG::Info::History& h, epoch_t from, vector& last); - Mutex snap_trimmer_lock; - Cond snap_trimmer_cond; - - void wake_snap_trimmer(); - void snap_trimmer(); // thread entry - - struct SnapTrimmer : public Thread { - OSD *osd; - SnapTrimmer(OSD *o) : osd(o) {} - void *entry() { - osd->snap_trimmer(); - return NULL; - } - } snap_trimmer_thread; - void wake_pg_waiters(pg_t pgid) { if (waiting_for_pg.count(pgid)) { take_waiters(waiting_for_pg[pgid]); @@ -513,6 +497,32 @@ private: }; + // -- snap trimming -- + xlist snap_trim_queue; + + struct SnapTrimWQ : public WorkQueue { + OSD *osd; + SnapTrimWQ(OSD *o) : osd(o) {} + + void _enqueue(PG *pg) { + osd->snap_trim_queue.push_back(&pg->snap_trim_item); + } + void _dequeue(PG *pg) { + pg->snap_trim_item.remove_myself(); + } + PG * _dequeue() { + if (osd->snap_trim_queue.empty()) + return NULL; + PG *pg = osd->snap_trim_queue.front(); + osd->snap_trim_queue.pop_front(); + return pg; + } + void _process(PG *pg) { + pg->snap_trimmer(); + } + } snap_trim_wq; + + // -- scrubbing -- xlist scrub_queue; diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 0091fc0cf6f4a..385979c1ef2eb 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -1300,11 +1300,7 @@ void PG::queue_snap_trim() state_set(PG_STATE_SNAPTRIMQUEUE); - osd->snap_trimmer_lock.Lock(); - osd->pgs_pending_snap_removal.push_back(&pending_snap_removal_item); - osd->snap_trimmer_lock.Unlock(); - - osd->wake_snap_trimmer(); // FIXME: we probably want to wait until at least peering completes? + osd->snap_trim_wq.queue(this); } diff --git a/src/osd/PG.h b/src/osd/PG.h index db3281a693583..8aa5f5d3379ac 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -556,15 +556,13 @@ public: set snap_collections; map past_intervals; - xlist::item recovery_item, scrub_item; + xlist::item recovery_item, scrub_item, snap_trim_item; int recovery_ops_active; protected: int role; // 0 = primary, 1 = replica, -1=none. int state; // see bit defns above - xlist::item pending_snap_removal_item; - // primary state public: vector acting; @@ -683,11 +681,10 @@ public: _lock("PG::_lock"), ref(0), deleted(false), info(p), - recovery_item(this), scrub_item(this), + recovery_item(this), scrub_item(this), snap_trim_item(this), recovery_ops_active(0), role(0), state(0), - pending_snap_removal_item(this), have_master_log(true), must_notify_mon(false), pg_stats_lock("PG::pg_stats_lock"), -- 2.39.5