mds: CDir: create scrub_info_t and surrounding scrub infrastructure

This is set up so that the coming-shortly scrub system can initialize scrubbing
on a directory and perform the actual scrubs, but rely on the directory to
keep track of its internal scrub state and selection of dentries.

Note that initialize_scrub() and scrub_local()  can only be called while the
directory is complete.

Signed-off-by: Greg Farnum <gfarnum@redhat.com>

diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index f923f18cb8c34011ba5e009602f4bf9a2a85a040..a3cfdf6ace2f3b0a6ab620ac536afd3231479111 100644 (file)
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -185,6 +185,7 @@ CDir::CDir(CInode *in, frag_t fg, MDCache *mdcache, bool auth) :
   first(2),
   dirty_rstat_inodes(member_offset(CInode, dirty_rstat_item)),
   projected_version(0),  item_dirty(this), item_new(this),
+  scrub_infop(NULL),
   num_head_items(0), num_head_null(0),
   num_snap_items(0), num_snap_null(0),
   num_dirty(0), committing_version(0), committed_version(0),
@@ -1263,6 +1264,11 @@ fnode_t *CDir::project_fnode()
   fnode_t *p = new fnode_t;
   *p = *get_projected_fnode();
   projected_fnode.push_back(p);
+  if (scrub_infop && scrub_infop->last_scrub_dirty) {
+    p->recursive_scrub_stamp = scrub_infop->last_recursive.time;
+    p->recursive_scrub_version = scrub_infop->last_recursive.version;
+    scrub_infop->last_scrub_dirty = false;
+    scrub_maybe_delete_info();
   dout(10) << "project_fnode " << p << dendl;
   return p;
 }
@@ -2848,3 +2854,170 @@ void CDir::dump(Formatter *f) const
   MDSCacheObject::dump(f);
 }
 
+/****** Scrub Stuff *******/
+
+void CDir::scrub_info_create() const
+{
+  assert(!scrub_infop);
+
+  // break out of const-land to set up implicit initial state
+  CDir *me = const_cast<CDir*>(this);
+  fnode_t *fn = me->get_projected_fnode();
+
+  scrub_info_t *si = new scrub_info_t();
+
+  si->last_recursive.version = si->recursive_start.version =
+      fn->recursive_scrub_version;
+  si->last_recursive.time = si->recursive_start.time =
+      fn->recursive_scrub_stamp;
+
+  me->scrub_infop = si;
+}
+
+void CDir::scrub_initialize()
+{
+  dout(20) << __func__ << dendl;
+  assert(is_complete());
+
+  // FIXME: weird implicit construction, is someone else meant
+  // to be calling scrub_info_create first?
+  scrub_info();
+  assert(scrub_infop && !scrub_infop->directory_scrubbing);
+
+  scrub_infop->recursive_start.version = get_projected_version();
+  scrub_infop->recursive_start.time = ceph_clock_now(g_ceph_context);
+
+  scrub_infop->directories_to_scrub.clear();
+  scrub_infop->directories_scrubbing.clear();
+  scrub_infop->directories_scrubbed.clear();
+  scrub_infop->others_to_scrub.clear();
+  scrub_infop->others_scrubbing.clear();
+  scrub_infop->others_scrubbed.clear();
+
+  for (map_t::iterator i = items.begin();
+      i != items.end();
+      ++i) {
+    // TODO: handle snapshot scrubbing
+    if (i->first.snapid != CEPH_NOSNAP)
+      continue;
+
+    CInode *in = i->second->get_projected_linkage()->get_inode();
+    if (in && in->is_dir())
+      scrub_infop->directories_to_scrub.insert(i->first);
+    else if (in)
+      scrub_infop->others_to_scrub.insert(i->first);
+  }
+  scrub_infop->directory_scrubbing = true;
+}
+
+void CDir::scrub_finished()
+{
+  dout(20) << __func__ << dendl;
+  assert(scrub_infop && scrub_infop->directory_scrubbing);
+
+  assert(scrub_infop->directories_to_scrub.empty());
+  assert(scrub_infop->directories_scrubbing.empty());
+  scrub_infop->directories_scrubbed.clear();
+  assert(scrub_infop->others_to_scrub.empty());
+  assert(scrub_infop->others_scrubbing.empty());
+  scrub_infop->others_scrubbed.clear();
+  scrub_infop->directory_scrubbing = false;
+
+  scrub_infop->last_scrub_dirty = true;
+
+  scrub_infop->last_recursive = scrub_infop->recursive_start;
+  scrub_infop->last_scrub_dirty = true;
+}
+
+int CDir::_next_dentry_on_set(set<dentry_key_t>& dns, bool missing_okay,
+                              MDSInternalContext *cb, CDentry **dnout)
+{
+  dentry_key_t dnkey;
+
+  while (!dns.empty()) {
+    set<dentry_key_t>::iterator front = dns.begin();
+    dnkey = *front;
+    *dnout = lookup(dnkey.name);
+    if (!*dnout) {
+      if (!is_complete() &&
+          (!has_bloom() || is_in_bloom(dnkey.name))) {
+        // need to re-read this dirfrag
+        fetch(cb);
+        return EAGAIN;
+      }
+      // okay, we lost it
+      if (missing_okay) {
+       dout(15) << " we no longer have directory dentry "
+                << dnkey.name << ", assuming it got renamed" << dendl;
+       dns.erase(dnkey);
+       continue;
+      } else {
+       dout(5) << " we lost dentry " << dnkey.name
+               << ", bailing out because that's impossible!" << dendl;
+       assert(0);
+      }
+    }
+    // okay, we got a  dentry
+    dns.erase(dnkey);
+
+    return 0;
+  }
+  *dnout = NULL;
+  return ENOENT;
+}
+
+int CDir::scrub_dentry_next(MDSInternalContext *cb, CDentry **dnout)
+{
+  dout(20) << __func__ << dendl;
+  assert(scrub_infop && scrub_infop->directory_scrubbing);
+
+  dout(20) << "trying to scrub directories underneath us" << dendl;
+  int rval = _next_dentry_on_set(scrub_infop->directories_to_scrub, true,
+                                 cb, dnout);
+  if (rval == 0) {
+    dout(20) << __func__ << " inserted to directories scrubbing: "
+      << *dnout << dendl;
+    scrub_infop->directories_scrubbing.insert((*dnout)->key());
+  } else if (rval < 0 || rval == EAGAIN) {
+    // we don't need to do anything else
+  } else { // we emptied out the directory scrub set
+    assert(rval == ENOENT);
+    dout(20) << "no directories left, moving on to other kinds of dentries"
+             << dendl;
+    
+    rval = _next_dentry_on_set(scrub_infop->others_to_scrub, false, cb, dnout);
+    if (rval == 0) {
+      dout(20) << __func__ << " inserted to others scrubbing: "
+        << *dnout << dendl;
+      scrub_infop->others_scrubbing.insert((*dnout)->key());
+    }
+  }
+  dout(20) << " returning " << rval << " with dn=" << *dnout << dendl;
+  return rval;
+}
+
+void CDir::scrub_dentry_finished(CDentry *dn)
+{
+  dout(20) << __func__ << " on dn " << *dn << dendl;
+  assert(scrub_infop && scrub_infop->directory_scrubbing);
+  dentry_key_t dn_key = dn->key();
+  if (scrub_infop->directories_scrubbing.count(dn_key)) {
+    scrub_infop->directories_scrubbing.erase(dn_key);
+    scrub_infop->directories_scrubbed.insert(dn_key);
+  } else {
+    assert(scrub_infop->others_scrubbing.count(dn_key));
+    scrub_infop->others_scrubbing.erase(dn_key);
+    scrub_infop->others_scrubbed.insert(dn_key);
+  }
+}
+
+void CDir::scrub_maybe_delete_info()
+{
+  if (scrub_infop &&
+      !scrub_infop->directory_scrubbing &&
+      !scrub_infop->last_scrub_dirty &&
+      scrub_infop->dirty_scrub_stamps.empty()) {
+    delete scrub_infop;
+    scrub_infop = NULL;
+  }
+}

diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index be0f10aa6b97919d6dee8c096a8cb8245b8fc1dd..c014fa467ea5adee6522788e525d0ea20df53971 100644 (file)
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -233,7 +233,89 @@ private:
 
 public:
   typedef std::map<dentry_key_t, CDentry*> map_t;
+
+  class scrub_info_t {
+  public:
+    /// inodes we contain with dirty scrub stamps
+    map<dentry_key_t,CInode*> dirty_scrub_stamps; // TODO: make use of this!
+    struct scrub_stamps {
+      version_t version;
+      utime_t time;
+      scrub_stamps() : version(0) {}
+      void operator=(const scrub_stamps &o) {
+        version = o.version;
+        time = o.time;
+      }
+    };
+
+    scrub_stamps recursive_start; // when we last started a recursive scrub
+    scrub_stamps last_recursive; // when we last finished a recursive scrub
+    bool directory_scrubbing; /// safety check
+    bool last_scrub_dirty; /// is scrub info dirty or is it flushed to fnode?
+
+    /// these are lists of children in each stage of scrubbing
+    set<dentry_key_t> directories_to_scrub;
+    set<dentry_key_t> directories_scrubbing;
+    set<dentry_key_t> directories_scrubbed;
+    set<dentry_key_t> others_to_scrub;
+    set<dentry_key_t> others_scrubbing;
+    set<dentry_key_t> others_scrubbed;
+
+    scrub_info_t() : directory_scrubbing(false), last_scrub_dirty(false) {}
+  };
+  /**
+   * Call to start this CDir on a new scrub.
+   * @pre It is not currently scrubbing
+   * @pre The CDir is marked complete.
+   * @post It has set up its internal scrubbing state.
+   */
+  void scrub_initialize();
+  /**
+   * Get the next dentry to scrub. Gives you a CDentry* and its meaning. This
+   * function will give you all directory-representing dentries before any
+   * others.
+   * 0: success, you should scrub this CDentry right now
+   * EAGAIN: is currently fetching the next CDentry into memory for you.
+   *   It will activate your callback when done; try again when it does!
+   * ENOENT: there are no remaining dentries to scrub
+   * <0: There was an unexpected error
+   *
+   * @param cb An MDSInternalContext which will be activated only if
+   *   we return EAGAIN via rcode, or else ignored
+   * @param dnout CDentry * which you should next scrub, or NULL
+   * @returns a value as described above
+   */
+  int scrub_dentry_next(MDSInternalContext *cb, CDentry **dnout);
+  /**
+   * Report to the CDir that a CDentry has been scrubbed. Call this
+   * for every CDentry returned from scrub_dentry_next().
+   * @param dn The CDentry which has been scrubbed.
+   */
+  void scrub_dentry_finished(CDentry *dn);
+  /**
+   * Call this once all CDentries have been scrubbed, according to
+   * scrub_dentry_next's listing. It finalizes the scrub statistics.
+   */
+  void scrub_finished();
+private:
+  /**
+   * Create a scrub_info_t struct for the scrub_infop pointer.
+   */
+  void scrub_info_create() const;
+  /**
+   * Delete the scrub_infop if it's not got any useful data.
+   */
+  void scrub_maybe_delete_info();
+  /**
+   * Check the given set (presumably one of those in scrub_info_t) for the
+   * next key to scrub and look it up (or fail!).
+   */
+  int _next_dentry_on_set(set<dentry_key_t>& dns, bool missing_okay,
+                          MDSInternalContext *cb, CDentry **dnout);
+
+
 protected:
+  scrub_info_t *scrub_infop;
 
   // contents of this directory
   map_t items;       // non-null AND null
@@ -297,11 +379,18 @@ protected:
  public:
   CDir(CInode *in, frag_t fg, MDCache *mdcache, bool auth);
   ~CDir() {
+    delete scrub_infop;
     remove_bloom();
     g_num_dir--;
     g_num_dirs++;
   }
 
+  const scrub_info_t *scrub_info() const {
+    if (!scrub_infop) {
+      scrub_info_create();
+    }
+    return scrub_infop;
+  }
 
 
   // -- accessors --