mds: add mds_debug_subtrees option

Verify that replayed ESubtreeMap events match what we have.

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/src/common/config.cc b/src/common/config.cc
index 597bcbe6d454983ce8bd3df10f80ca6ec2e6d343..5b5194a1912acc54153144ba024eb4b9b9d8af70 100644 (file)
--- a/src/common/config.cc
+++ b/src/common/config.cc
@@ -308,6 +308,7 @@ struct config_option config_optionsp[] = {
   OPTION(mds_debug_scatterstat, OPT_BOOL, false),
   OPTION(mds_debug_frag, OPT_BOOL, false),
   OPTION(mds_debug_auth_pins, OPT_BOOL, false),
+  OPTION(mds_debug_subtrees, OPT_BOOL, false),
   OPTION(mds_kill_mdstable_at, OPT_INT, 0),
   OPTION(mds_kill_export_at, OPT_INT, 0),
   OPTION(mds_kill_import_at, OPT_INT, 0),

diff --git a/src/common/config.h b/src/common/config.h
index 5468e1074337c11003cdbdda8591aa08c602ee63..54608e9a401399282dbafeefd2aae9850f933cf0 100644 (file)
--- a/src/common/config.h
+++ b/src/common/config.h
@@ -373,6 +373,7 @@ public:
   bool mds_debug_scatterstat;
   bool mds_debug_frag;
   bool mds_debug_auth_pins;
+  bool mds_debug_subtrees;
   int mds_kill_mdstable_at;
   int mds_kill_export_at;
   int mds_kill_import_at;

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index fcbc17759d4e2d8a05bb527c77d151a6062208d8..e102779669503058174424ba095c1bbdb615095c 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -568,6 +568,9 @@ public:
     return subtrees[dir].empty();
   }
   void remove_subtree(CDir *dir);
+  bool is_subtree(CDir *root) {
+    return subtrees.count(root);
+  }
   void get_subtree_bounds(CDir *root, set<CDir*>& bounds);
   void get_wouldbe_subtree_bounds(CDir *root, set<CDir*>& bounds);
   void verify_subtree_bounds(CDir *root, const set<CDir*>& bounds);

diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index 64d7c4f8af219770fbf721b6f8267ee77e475942..6fe73f46163aea5f57453f3d6918bc38455a964b 100644 (file)
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -1099,7 +1099,92 @@ void ESubtreeMap::replay(MDS *mds)
 
   // suck up the subtree map?
   if (mds->mdcache->is_subtrees()) {
-    dout(10) << "ESubtreeMap.replay -- ignoring, already have import map" << dendl;
+    dout(10) << "ESubtreeMap.replay -- i already have import map; verifying" << dendl;
+    int errors = 0;
+
+    for (map<dirfrag_t, vector<dirfrag_t> >::iterator p = subtrees.begin();
+        p != subtrees.end();
+        ++p) {
+      CDir *dir = mds->mdcache->get_dirfrag(p->first);
+      if (!dir) {
+       mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+                         << " subtree root " << p->first << " not in cache";
+       ++errors;
+       continue;
+      }
+      
+      if (!mds->mdcache->is_subtree(dir)) {
+       mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+                         << " subtree root " << p->first << " not a subtree in cache";
+       ++errors;
+       continue;
+      }
+      if (dir->get_dir_auth().first != mds->whoami) {
+       mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+                         << " subtree root " << p->first
+                         << " is not mine in cache (it's " << dir->get_dir_auth() << ")";
+       ++errors;
+       continue;
+      }
+
+      set<CDir*> bounds;
+      mds->mdcache->get_subtree_bounds(dir, bounds);
+      for (vector<dirfrag_t>::iterator q = p->second.begin(); q != p->second.end(); ++q) {
+       CDir *b = mds->mdcache->get_dirfrag(*q);
+       if (!b) {
+         mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+                           << " subtree " << p->first << " bound " << *q << " not in cache";
+       ++errors;
+         continue;
+       }
+       if (bounds.count(b) == 0) {
+         mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+                           << " subtree " << p->first << " bound " << *q << " not a bound in cache";
+       ++errors;
+         continue;
+       }
+       bounds.erase(b);
+      }
+      for (set<CDir*>::iterator q = bounds.begin(); q != bounds.end(); ++q) {
+       mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+                         << " subtree " << p->first << " has extra bound in cache " << (*q)->dirfrag();
+       ++errors;
+      }
+      
+      if (ambiguous_subtrees.count(p->first)) {
+       if (!mds->mdcache->have_ambiguous_import(p->first)) {
+         mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+                           << " subtree " << p->first << " is ambiguous but is not in our cache";
+         ++errors;
+       }
+      } else {
+       if (mds->mdcache->have_ambiguous_import(p->first)) {
+         mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+                           << " subtree " << p->first << " is not ambiguous but is in our cache";
+         ++errors;
+       }
+      }
+    }
+    
+    list<CDir*> subs;
+    mds->mdcache->list_subtrees(subs);
+    for (list<CDir*>::iterator p = subs.begin(); p != subs.end(); ++p) {
+      CDir *dir = *p;
+      if (dir->get_dir_auth().first != mds->whoami)
+       continue;
+      if (subtrees.count(dir->dirfrag()) == 0) {
+       mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+                         << " does not include cache subtree " << dir->dirfrag();
+       ++errors;
+      }
+    }
+
+    if (errors) {
+      dout(0) << "journal subtrees: " << subtrees << dendl;
+      dout(0) << "journal ambig_subtrees: " << ambiguous_subtrees << dendl;
+      mds->mdcache->show_subtrees();
+      assert(!g_conf->mds_debug_subtrees || errors == 0);
+    }
     return;
   }
 

diff --git a/src/vstart.sh b/src/vstart.sh
index 906b3f4e293e19f1873debd62190d555ddb1e980..d8fac39fba65b60b1098c4104897504721fdf4ce 100755 (executable)
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -279,6 +279,7 @@ $DAEMONOPTS
 $CMDSDEBUG
         mds debug frag = true
         mds debug auth pins = true
+        mds debug subtrees = true
 [osd]
 $DAEMONOPTS
         osd class tmp = out