From: Greg Farnum Date: Wed, 1 Apr 2015 22:02:28 +0000 (-0700) Subject: mds: initial ScrubStack implementation X-Git-Tag: v10.0.1~51^2~15 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=be04a9004d3fa0a9bde8e637e6824f70c2cec627;p=ceph.git mds: initial ScrubStack implementation Signed-off-by: Greg Farnum --- diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 1f4727b35b92..cce3fee832b5 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -507,6 +507,8 @@ OPTION(mds_max_purge_ops_per_pg, OPT_FLOAT, 0.5) OPTION(mds_root_ino_uid, OPT_INT, 0) // The UID of / on new filesystems OPTION(mds_root_ino_gid, OPT_INT, 0) // The GID of / on new filesystems +OPTION(mds_max_scrub_ops_in_progress, OPT_INT, 5) // the number of simultaneous scrubs allowed + // If true, compact leveldb store on mount OPTION(osd_compact_leveldb_on_mount, OPT_BOOL, false) diff --git a/src/mds/CDentry.h b/src/mds/CDentry.h index 424b8fdd6042..7470ff4db6ad 100644 --- a/src/mds/CDentry.h +++ b/src/mds/CDentry.h @@ -165,6 +165,7 @@ protected: public: elist::item item_dirty; elist::item item_stray; + elist::item item_scrub; const scrub_info_t *scrub_info() const { if(!scrub_infop) @@ -230,6 +231,7 @@ public: } ~CDentry() { assert(!scrub_infop); + assert(!item_scrub.is_on_list()); g_num_dn--; g_num_dns++; } diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index d1fe9474e45a..bc657540c0f6 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -19,8 +19,6 @@ #include "messages/MMDSMap.h" #include "MDSMap.h" -//#include "MDS.h" -#include "mds_table_types.h" #include "SnapClient.h" #include "SnapServer.h" #include "MDBalancer.h" @@ -30,6 +28,7 @@ #include "InoTable.h" #include "mon/MonClient.h" #include "common/HeartbeatMap.h" +#include "ScrubStack.h" #include "MDSRank.h" @@ -56,7 +55,8 @@ MDSRank::MDSRank( mdsmap(mdsmap_), objecter(objecter_), server(NULL), mdcache(NULL), locker(NULL), mdlog(NULL), - balancer(NULL), inotable(NULL), snapserver(NULL), snapclient(NULL), + balancer(NULL), scrubstack(NULL), + inotable(NULL), snapserver(NULL), snapclient(NULL), sessionmap(this), logger(NULL), mlogger(NULL), op_tracker(g_ceph_context, g_conf->mds_enable_op_tracker, g_conf->osd_num_op_tracker_shard), @@ -79,6 +79,8 @@ MDSRank::MDSRank( mdlog = new MDLog(this); balancer = new MDBalancer(this, messenger, monc); + scrubstack = new ScrubStack(mdcache, finisher); + inotable = new InoTable(this); snapserver = new SnapServer(this, monc); snapclient = new SnapClient(this); @@ -98,6 +100,7 @@ MDSRank::~MDSRank() g_ceph_context->get_heartbeat_map()->remove_worker(hb); } + if (scrubstack) { delete scrubstack; scrubstack = NULL; } if (mdcache) { delete mdcache; mdcache = NULL; } if (mdlog) { delete mdlog; mdlog = NULL; } if (balancer) { delete balancer; balancer = NULL; } diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index 96b0688f050c..6dc59fe3ca8a 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -107,6 +107,7 @@ class Objecter; class MonClient; class Finisher; class MMDSMap; +class ScrubStack; /** * The public part of this class's interface is what's exposed to all @@ -151,6 +152,7 @@ class MDSRank { Locker *locker; MDLog *mdlog; MDBalancer *balancer; + ScrubStack *scrubstack; InoTable *inotable; diff --git a/src/mds/Makefile-server.am b/src/mds/Makefile-server.am index ee3daed88937..1de7c5465ead 100644 --- a/src/mds/Makefile-server.am +++ b/src/mds/Makefile-server.am @@ -36,6 +36,7 @@ noinst_HEADERS += \ mds/Mutation.h \ mds/Migrator.h \ mds/ScatterLock.h \ + mds/ScrubStack.h \ mds/Server.h \ mds/SessionMap.h \ mds/SimpleLock.h \ diff --git a/src/mds/Makefile.am b/src/mds/Makefile.am index c7b03076c7b0..021a3fcb7449 100644 --- a/src/mds/Makefile.am +++ b/src/mds/Makefile.am @@ -23,6 +23,7 @@ LIBMDS_SOURCES = \ mds/MDSTableClient.cc \ mds/MDSTableServer.cc \ mds/SimpleLock.cc \ + mds/ScrubStack.cc \ mds/SnapRealm.cc \ mds/SnapServer.cc \ mds/snap.cc \ diff --git a/src/mds/ScrubStack.cc b/src/mds/ScrubStack.cc new file mode 100644 index 000000000000..67029c6bb922 --- /dev/null +++ b/src/mds/ScrubStack.cc @@ -0,0 +1,430 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014 Red Hat + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include + +#include "ScrubStack.h" +#include "common/Finisher.h" +#include "mds/MDSRank.h" +#include "mds/MDCache.h" +#include "mds/MDSContinuation.h" + +#define dout_subsys ceph_subsys_mds +#undef dout_prefix +#define dout_prefix _prefix(_dout, scrubstack->mdcache->mds) +static ostream& _prefix(std::ostream *_dout, MDSRank *mds) { + return *_dout << "mds." << mds->get_nodeid() << ".scrubstack "; +} + +void ScrubStack::push_dentry(CDentry *dentry) +{ + dout(20) << "pushing " << *dentry << " on top of ScrubStack" << dendl; + dentry->get(CDentry::PIN_SCRUBQUEUE); + if (!dentry->item_scrub.is_on_list()) { + stack_size++; + } + dentry_stack.push_front(&dentry->item_scrub); +} + +void ScrubStack::push_dentry_bottom(CDentry *dentry) +{ + dout(20) << "pushing " << *dentry << " on bottom of ScrubStack" << dendl; + dentry->get(CDentry::PIN_SCRUBQUEUE); + if (!dentry->item_scrub.is_on_list()) { + stack_size++; + } + dentry_stack.push_back(&dentry->item_scrub); +} + +void ScrubStack::pop_dentry(CDentry *dn) +{ + dout(20) << "popping " << *dn + << " off of ScrubStack" << dendl; + assert(dn->item_scrub.is_on_list()); + dn->put(CDentry::PIN_SCRUBQUEUE); + dn->item_scrub.remove_myself(); + stack_size--; +} + +void ScrubStack::_enqueue_dentry(CDentry *dn, CDir *parent, bool recursive, + bool children, const std::string &tag, + MDSInternalContextBase *on_finish, bool top) +{ + dout(10) << __func__ << " with {" << *dn << "}" + << ", recursive=" << recursive << ", children=" << children + << ", on_finish=" << on_finish << ", top=" << top << dendl; + assert(mdcache->mds->mds_lock.is_locked_by_me()); + dn->scrub_initialize(parent, recursive, children, on_finish); + if (top) + push_dentry(dn); + else + push_dentry_bottom(dn); +} + +void ScrubStack::enqueue_dentry(CDentry *dn, bool recursive, bool children, + const std::string &tag, + MDSInternalContextBase *on_finish, bool top) +{ + _enqueue_dentry(dn, NULL, recursive, children, tag, on_finish, top); + kick_off_scrubs(); +} + +void ScrubStack::kick_off_scrubs() +{ + dout(20) << __func__ << " entering with " << scrubs_in_progress << " in " + "progress and " << stack_size << " in the stack" << dendl; + bool can_continue = true; + elist::iterator i = dentry_stack.begin(); + while (g_conf->mds_max_scrub_ops_in_progress > scrubs_in_progress && + can_continue && !i.end()) { + CDentry *cur = *i; + + dout(20) << __func__ << " examining dentry " << *cur << dendl; + + CInode *curi = cur->get_projected_inode(); + ++i; // we have our reference, push iterator forward + + if (!curi->is_dir()) { + // it's a regular file, symlink, or hard link + pop_dentry(cur); // we only touch it this once, so remove from stack + + if (curi->is_file()) { + scrub_file_dentry(cur); + can_continue = true; + } else { + // drat, we don't do anything with these yet :( + dout(5) << "skipping scrub on non-dir, non-file dentry " + << *cur << dendl; + } + } else { + bool completed; // it's done, so pop it off the stack + bool terminal; // not done, but we can start ops on other directories + bool progress; // it added new dentries to the top of the stack + scrub_dir_dentry(cur, &progress, &terminal, &completed); + if (completed) { + dout(20) << __func__ << " dir completed" << dendl; + pop_dentry(cur); + } else if (progress) { + dout(20) << __func__ << " dir progressed" << dendl; + // we added new stuff to top of stack, so reset ourselves there + i = dentry_stack.begin(); + } else { + dout(20) << __func__ << " dir no-op" << dendl; + } + + can_continue = progress || terminal || completed; + } + } +} + +void ScrubStack::scrub_dir_dentry(CDentry *dn, + bool *added_children, + bool *terminal, + bool *done) +{ + assert(dn != NULL); + dout(10) << __func__ << *dn << dendl; + + if (!dn->scrub_info()->scrub_children && + !dn->scrub_info()->scrub_recursive) { + // TODO: we have to scrub the local dentry/inode, but nothing else + } + + *added_children = false; + *terminal = false; + *done = false; + + CInode *in = dn->get_projected_inode(); + // FIXME: greg -- is get_version the appropriate version? (i.e. is scrub_version + // meant to be an actual version that we're scrubbing, or something else?) + if (!in->scrub_info()->scrub_in_progress) { + // We may come through here more than once on our way up and down + // the stack... or actually is that right? Should we perhaps + // only see ourselves once on the way down and once on the way + // back up again, and not do this? + + // Hmm, bigger problem, we can only actually + + in->scrub_initialize(in->get_version()); + } + + list scrubbing_frags; + list scrubbing_cdirs; + in->scrub_dirfrags_scrubbing(&scrubbing_frags); + dout(20) << __func__ << " iterating over " << scrubbing_frags.size() + << " scrubbing frags" << dendl; + for (list::iterator i = scrubbing_frags.begin(); + i != scrubbing_frags.end(); + ++i) { + // turn frags into CDir * + CDir *dir = in->get_dirfrag(*i); + scrubbing_cdirs.push_back(dir); + dout(25) << __func__ << " got CDir " << *dir << " presently scrubbing" << dendl; + } + + + dout(20) << __func__ << " consuming from " << scrubbing_cdirs.size() + << " scrubbing cdirs" << dendl; + + list::iterator i = scrubbing_cdirs.begin(); + bool all_frags_terminal = true; + bool all_frags_done = true; + bool finally_done = false; + while (g_conf->mds_max_scrub_ops_in_progress > scrubs_in_progress) { + // select next CDir + CDir *cur_dir = NULL; + if (i != scrubbing_cdirs.end()) { + cur_dir = *i; + ++i; + dout(20) << __func__ << " got cur_dir = " << *cur_dir << dendl; + } else { + bool ready = get_next_cdir(in, &cur_dir); + dout(20) << __func__ << " get_next_cdir ready=" << ready << dendl; + if (cur_dir) { + cur_dir->scrub_initialize(); + } + + if (ready && cur_dir) { + scrubbing_cdirs.push_back(cur_dir); + } else if (!ready) { + // We are waiting for load of a frag + all_frags_done = false; + all_frags_terminal = false; + break; + } else { + // Finished with all frags + break; + } + } + // scrub that CDir + bool frag_added_children = false; + bool frag_terminal = true; + bool frag_done = false; + scrub_dirfrag(cur_dir, &frag_added_children, &frag_terminal, &frag_done); + if (frag_done) { + // FIXME is this right? Can we end up hitting this more than + // once and is that a problem? + cur_dir->inode->scrub_dirfrag_finished(cur_dir->frag); + } + *added_children |= frag_added_children; + all_frags_terminal = all_frags_terminal && frag_terminal; + all_frags_done = all_frags_done && frag_done; + } + + dout(20) << "finished looping; all_frags_terminal=" << all_frags_terminal + << ", all_frags_done=" << all_frags_done << dendl; + if (all_frags_done) { + assert (!*added_children); // can't do this if children are still pending + scrub_dir_dentry_final(dn, &finally_done); + } + + *terminal = all_frags_terminal; + *done = all_frags_done && finally_done; + dout(10) << __func__ << " is exiting " << *terminal << " " << *done << dendl; + return; +} + +bool ScrubStack::get_next_cdir(CInode *in, CDir **new_dir) +{ + dout(20) << __func__ << " on " << *in << dendl; + frag_t next_frag; + int r = in->scrub_dirfrag_next(&next_frag); + assert (r >= 0); + + if (r == 0) { + // we got a frag to scrub, otherwise it would be ENOENT + dout(25) << "looking up new frag " << next_frag << dendl; + CDir *next_dir = in->get_or_open_dirfrag(mdcache, next_frag); + if (!next_dir->is_complete()) { + C_KickOffScrubs *c = new C_KickOffScrubs(mdcache->mds, this); + next_dir->fetch(c); + dout(25) << "fetching frag from RADOS" << dendl; + return false; + } + *new_dir = next_dir; + dout(25) << "returning dir " << *new_dir << dendl; + return true; + } + assert(r == ENOENT); + // there are no dirfrags left + *new_dir = NULL; + return true; +} + +void ScrubStack::scrub_dir_dentry_final(CDentry *dn, bool *finally_done) +{ + dout(20) << __func__ << *dn << dendl; + *finally_done =true; + // FIXME: greg -- is this the right lifetime from the inode's scrub_info? + CInode *in = dn->get_projected_inode(); + + Context *fin = NULL; + in->scrub_finished(&fin); + if (fin) { + // FIXME: pass some error code in? + finisher->queue(new MDSIOContextWrapper(mdcache->mds, fin), 0); + } + return; +} + +void ScrubStack::scrub_dirfrag(CDir *dir, bool *added_children, + bool *is_terminal, bool *done) +{ + assert(dir != NULL); + + dout(20) << __func__ << " on " << *dir << dendl; + *added_children = false; + *is_terminal = false; + *done = false; + + // XXX HACK get the frag complete before calling + // scrub intiialize + if (!dir->is_complete()) { + dir->fetch(new C_KickOffScrubs(mdcache->mds, this)); + return; + } + + if (!dir->scrub_info()->directory_scrubbing) { + // FIXME: greg - the CDir API seems inconsistent here, as + // scrub_initialize wants the dir to be complete before + // we start, but scrub_dentry_next handles incompleteness + // via its EAGAIN path. + dir->scrub_initialize(); + } + + int r = 0; + while(r == 0) { + MDSInternalContext *kick = new C_KickOffScrubs(mdcache->mds, this); + CDentry *dn = NULL; + r = dir->scrub_dentry_next(kick, &dn); + if (r != EAGAIN) { + // ctx only used by scrub_dentry_next in EAGAIN case + // FIXME It's kind of annoying to keep allocating and deleting a ctx here + delete kick; + } + + if (r == EAGAIN) { + // Drop out, CDir fetcher will call back our kicker context + return; + } + + if (r == ENOENT) { + // Nothing left to scrub, are we done? + std::list scrubbing; + dir->scrub_dentries_scrubbing(&scrubbing); + if (scrubbing.empty()) { + // FIXME: greg: What's the diff meant to be between done and terminal + *done = true; + *is_terminal = true; + continue; + } else { + return; + } + } + + if (r < 0) { + // FIXME: how can I handle an error here? I can't hold someone up + // forever, but I can't say "sure you're scrubbed" + // -- should change scrub_dentry_next definition to never + // give out IO errors (handle them some other way) + // + derr << __func__ << " error from scrub_dentry_next: " + << r << dendl; + return; + } + + // scrub_dentry_next defined to only give -ve, EAGAIN, ENOENT, 0 -- we should + // never get random IO errors here. + assert(r == 0); + + // FIXME: Do I *really* need to construct a kick context for every + // single dentry I'm going to scrub? + MDSInternalContext *on_d_scrub = new C_KickOffScrubs(mdcache->mds, this); + _enqueue_dentry(dn, + dir, + /* + * FIXME: look up params from dir's inode's parent + */ +#if 0 + dir->scrub_info()->scrub_recursive, + dir->scrub_info()->scrub_children, +#else + true, + true, +#endif + // FIXME: carry tag through + "foobar", + on_d_scrub, + true); + + *added_children = true; + } + +#if 0 + bool any_unscrubbed = false; + + for (CDir::map_t::iterator i = dir->begin(); i != dir->end(); ++ i) { + CDentry *dn = i->second; + + if (dir->scrub_info()->directories_scrubbed.count(i->first) == 0 + && dir->scrub_info()->others_scrubbed.count(i->first) == 0) { + dout(20) << __func__ << " dn not yet scrubbed: " << *dn << dendl; + any_unscrubbed = true; + } + + if (dn->scrub_info()->dentry_scrubbing + || dir->scrub_info()->directories_scrubbed.count(i->first) + || dir->scrub_info()->others_scrubbed.count(i->first)) { + dout(20) << __func__ << " skipping already scrubbing or scrubbed " << *dn << dendl; + continue; + } + + C_KickOffScrubs *c = new C_KickOffScrubs(mdcache->mds, this); + + _enqueue_dentry(dn, + dir, + /* + * FIXME: look up params from dir's inode's parent + */ +#if 0 + dir->scrub_info()->scrub_recursive, + dir->scrub_info()->scrub_children, +#else + true, + true, +#endif + // FIXME: carry tag through + "foobar", + c, + true); + + *added_children = true; + } + + *done = !any_unscrubbed; +#endif +} + +void ScrubStack::scrub_file_dentry(CDentry *dn) +{ + // No-op: + // TODO: hook into validate_disk_state + Context *c = NULL; + dn->scrub_finished(&c); + if (c) { + // FIXME: pass some error code in? + finisher->queue(new MDSIOContextWrapper(mdcache->mds, c), 0); + } +} + diff --git a/src/mds/ScrubStack.h b/src/mds/ScrubStack.h new file mode 100644 index 000000000000..0be512646d70 --- /dev/null +++ b/src/mds/ScrubStack.h @@ -0,0 +1,186 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014 Red Hat + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef SCRUBSTACK_H_ +#define SCRUBSTACK_H_ + +#include "CDir.h" +#include "CDentry.h" +#include "CInode.h" +#include "MDSContext.h" + +#include "include/elist.h" + +class MDCache; +class Finisher; + +class ScrubStack { + protected: + /// A finisher needed so that we don't re-enter kick_off_scrubs + Finisher *finisher; + + /// The stack of dentries we want to scrub + elist dentry_stack; + /// current number of dentries we're actually scrubbing + int scrubs_in_progress; + ScrubStack *scrubstack; // hack for dout + int stack_size; +public: + MDCache *mdcache; + ScrubStack(MDCache *mdc, Finisher *finisher_) : + finisher(finisher_), + dentry_stack(member_offset(CDentry, item_scrub)), + scrubs_in_progress(0), + scrubstack(this), + stack_size(0), + mdcache(mdc) {} + ~ScrubStack() { + assert(dentry_stack.empty()); + assert(!scrubs_in_progress); + } + /** + * Put a dentry on the top of the scrub stack, so it is the highest priority. + * If there are other scrubs in progress, they will not continue scrubbing new + * entries until this one is completed. + * @param dn The dentry to scrub + * @param recursive True if we want to recursively scrub the + * entire hierarchy under dn. + * @param children True if we want to scrub the direct children of + * dn but aren't doing a recursive scrub. (Otherwise, all checks are + * local to dn's disk state.) + * @param tag If non-empty, tag applied to each verified rados object + */ + void enqueue_dentry_top(CDentry *dn, bool recursive, bool children, + const std::string &tag, + MDSInternalContextBase *on_finish) { + enqueue_dentry(dn, recursive, children, tag, on_finish, true); + } + /** Like enqueue_dentry_top, but we wait for all pending scrubs before + * starting this one. + */ + void enqueue_dentry_bottom(CDentry *dn, bool recursive, bool children, + const std::string &tag, + MDSInternalContextBase *on_finish) { + enqueue_dentry(dn, recursive, children, tag, on_finish, false); + } + +private: + /** + * Put the dentry at either the top or bottom of the stack, with + * the given scrub params, and then try and kick off more scrubbing. + */ + void enqueue_dentry(CDentry *dn, bool recursive, bool children, + const std::string &tag, + MDSInternalContextBase *on_finish, bool top); + void _enqueue_dentry(CDentry *dn, CDir *parent, bool recursive, bool children, + const std::string &tag, + MDSInternalContextBase *on_finish, bool top); + /** + * Kick off as many scrubs as are appropriate, based on the current + * state of the stack. + */ + void kick_off_scrubs(); + /** + * Push a dentry on top of the stack. + */ + inline void push_dentry(CDentry *dentry); + /** + * Push a dentry to the bottom of the stack. + */ + inline void push_dentry_bottom(CDentry *dentry); + /** + * Pop the given dentry off the stack. + */ + inline void pop_dentry(CDentry *dn); + + /** + * Scrub a file-representing dentry. + * @param dn The dentry to scrub + * @param progress Out pointer to a bool, which will be set to true. + * @pre dn->get_projected_inode()->is_file()==true; + */ + void scrub_file_dentry(CDentry *dn); + /** + * Make progress on scrubbing a directory-representing dirfrag and + * its children.. + * + * 1) Select the next dirfrag which hasn't been scrubbed, and make progress + * on it if possible. + * + * 2) If not, move on to the next dirfrag and start it up, if any. + * + * 3) If waiting for results from dirfrag scrubs, do nothing. + * + * 4) If all dirfrags have been scrubbed, scrub my inode. + * + * @param dn The CDentry to scrub as a directory + * @param added_dentries set to true if we pushed some of our children + * onto the ScrubStack + * @param is_terminal set to true if there are no descendant dentries + * remaining to start scrubbing. + * @param done set to true if we and all our children have finished scrubbing + */ + void scrub_dir_dentry(CDentry *dn, bool *added_children, bool *is_terminal, + bool *done); + /** + * Make progress on scrubbing a dirfrag. It may return after each of the + * following steps, but will report making progress on each one. + * + * 1) enqueues the next unscrubbed child directory dentry at the + * top of the stack. + * + * 2) Initiates a scrub on the next unscrubbed file dentry + * + * If there are scrubs currently in progress on child dentries, no more child + * dentries to scrub, and this function is invoked, it will report no + * progress. Try again later. + * + */ + void scrub_dirfrag(CDir *dir, bool *added_children, bool *is_terminal, + bool *done); + /** + * Scrub a directory-representing dentry. + * + * @param dn The CDentry of the directory we're doing final scrub on. + * @param done Set to true if the dentry scrub is finished. (That + * won't every actually happen, right? It needs disk accesses? I forget.) + */ + void scrub_dir_dentry_final(CDentry *dn, bool *done); + + /** + * Get a CDir into memory, and return it if it's already complete. + * Otherwise, fetch it and kick off scrubbing when done. + * + * @param in The Inode to get the next directory from + * @param new_dir The CDir we're returning to you. NULL if + * not ready yet or there aren't any. + * @returns false if you have to wait, true if there's no work + * left to do (we returned it, or there are none left in this inode). + */ + bool get_next_cdir(CInode *in, CDir **new_dir); + + class C_KickOffScrubs : public MDSInternalContext { + ScrubStack *stack; + public: + C_KickOffScrubs(MDSRank *mds, ScrubStack *s) + : MDSInternalContext(mds), stack(s) + { + } + void finish(int r){ + stack->kick_off_scrubs(); + } + }; +}; + +#endif /* SCRUBSTACK_H_ */