MDCache: make scrub_dentry schedulable and reentrant

Rather than assuming that any necessary inodes are in the cache, split up
MDCache::scrub_dentry into setup and work phases. Add an internal_op_finisher()
to MDRequest. Dispatch any CEPH_MDS_OP_VALIDATE internal operations to
scrub_dentry_work(). Taken together, these make everything work properly when
path_traverse() (by way of rdlock_path_pin_ref()) needs to go to disk before
satisfying the lookup.

Signed-off-by: Greg Farnum <greg@inktank.com>

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 492fc511f513604e5bc021ac9aa96e1519ed635f..7ed7553830521079ca41e3f948154729ed3fb501 100644 (file)
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -3684,6 +3684,15 @@ void CInode::validated_data::dump(Formatter *f) const
       f->dump_stream("error_str") << raw_rstats.error_str;
     }
     f->close_section(); // raw_rstats
+    // dump failure return code
+    int rc = 0;
+    if (backtrace.checked && backtrace.ondisk_read_retval)
+      rc = backtrace.ondisk_read_retval;
+    if (inode.checked && inode.ondisk_read_retval)
+      rc = inode.ondisk_read_retval;
+    if (raw_rstats.checked && raw_rstats.ondisk_read_retval)
+      rc = raw_rstats.ondisk_read_retval;
+    f->dump_int("return_code", rc);
   }
   f->close_section(); // results
 }

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 0d44e36678caf651d67fbbbdbee136aa3a894a0d..c296fd4c05952d443eea2e485c81e8516693c99c 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -8482,6 +8482,9 @@ void MDCache::dispatch_request(MDRequestRef& mdr)
     case CEPH_MDS_OP_EXPORTDIR:
       migrator->dispatch_export_dir(mdr);
       break;
+    case CEPH_MDS_OP_VALIDATE:
+      scrub_dentry_work(mdr);
+      break;
     default:
       assert(0);
     }
@@ -11509,7 +11512,13 @@ public:
     mdcache(mdc), mdr(mdr), on_finish(fin), formatter(f) {}
 
   void finish(int r) {
-    CInode::dump_validation_results(results, formatter);
+    if (r >= 0) { // we got into the scrubbing dump it
+      results.dump(formatter);
+    } else { // we failed the lookup or something; dump ourselves
+      formatter->open_object_section("results");
+      formatter->dump_int("return_code", r);
+      formatter->close_section(); // results
+    }
     mdcache->request_finish(mdr);
     on_finish->complete(r);
   }
@@ -11519,20 +11528,31 @@ void MDCache::scrub_dentry(const string& path, Formatter *f, Context *fin)
 {
   dout(10) << "scrub_dentry " << path << dendl;
   MDRequestRef mdr = request_start_internal(CEPH_MDS_OP_VALIDATE);
-  set<SimpleLock*> rdlocks, wrlocks, xlocks;
   filepath fp(path.c_str());
   mdr->set_filepath(fp);
+  C_scrub_dentry_finish *csd = new C_scrub_dentry_finish(this, mdr, fin, f);
+  mdr->internal_op_finish = csd;
+  mdr->internal_op_private = &csd->results;
+  scrub_dentry_work(mdr);
+}
 
-  // TODO: this is not actually safe with multiple MDSes!
+void MDCache::scrub_dentry_work(MDRequestRef& mdr)
+{
+  set<SimpleLock*> rdlocks, wrlocks, xlocks;
   CInode *in = mds->server->rdlock_path_pin_ref(mdr, 0, rdlocks, true);
-  assert(in != NULL); // TODO: obviously can't keep this, but it helps ensure authness
+  if (NULL == in)
+    return;
+
+  // TODO: Remove this restriction
   assert(in->is_auth());
 
   bool locked = mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks);
   if (!locked)
     return;
-  C_scrub_dentry_finish *finisher = new C_scrub_dentry_finish(this, mdr,
-                                                              fin, f);
-  in->validate_disk_state(&finisher->results, finisher);
+
+  CInode::validated_data *vr =
+      static_cast<CInode::validated_data*>(mdr->internal_op_private);
+
+  in->validate_disk_state(vr, mdr->internal_op_finish);
   return;
 }

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index bd991f618a8ab8aff9e426f38430b535ddc9f3a0..d83b5aa8deebedad231c0599e062fc3cce7e61d0 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -1033,6 +1033,7 @@ public:
     return p->second;
   }
   void scrub_dentry(const string& path, Formatter *f, Context *fin);
+  void scrub_dentry_work(MDRequestRef& mdr);
 };
 
 class C_MDS_RetryRequest : public MDSInternalContext {

diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 166fa3bfa10740aa6d8a388ff938dac6cd272ed0..a90ea19c464f21bfaa5cfea11769c34d3d08feb5 100644 (file)
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -213,6 +213,8 @@ struct MDRequestImpl : public MutationImpl, public TrackedOp {
 
   // -- i am an internal op
   int internal_op;
+  Context *internal_op_finish;
+  void *internal_op_private;
 
   // indicates how may retries of request have been made
   int retry;
@@ -299,7 +301,9 @@ struct MDRequestImpl : public MutationImpl, public TrackedOp {
     client_request(params.client_req), straydn(NULL), snapid(CEPH_NOSNAP),
     tracei(NULL), tracedn(NULL), alloc_ino(0), used_prealloc_ino(0), snap_caps(0),
     did_early_reply(false), o_trunc(false), getattr_caps(0),
-    slave_request(NULL), internal_op(params.internal_op), retry(0),
+    slave_request(NULL), internal_op(params.internal_op), internal_op_finish(NULL),
+    internal_op_private(NULL),
+    retry(0),
     waited_for_osdmap(false), _more(NULL) {
     in[0] = in[1] = NULL;
     if (!params.throttled.is_zero())