mds: flush after scrub repairs

Otherwise, if we restart the MDS right after the scrub,
then scrub again, it will see the same inconsistency when
it looks at the on-disk state.

This involves adapting the use of ScrubHeader to be something
writeable during scrub to record whether repair happened.

Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index f1974c54ce842fec5e3e86fd45c3bd89ae204b0a..f12600b8afc98b0b5487c15abd69883d514e360d 100644 (file)
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -3955,7 +3955,12 @@ next:
                                        << "(" << path << "), rewriting it";
         in->_mark_dirty_parent(in->mdcache->mds->mdlog->get_current_segment(),
                            false);
+        // Flag that we repaired this BT so that it won't go into damagetable
         results->backtrace.repaired = true;
+
+        // Flag that we did some repair work so that our repair operation
+        // can be flushed at end of scrub
+        in->scrub_infop->header->set_repaired();
       }
 
       // If the inode's number was free in the InoTable, fix that
@@ -4320,7 +4325,7 @@ void CInode::scrub_maybe_delete_info()
 }
 
 void CInode::scrub_initialize(CDentry *scrub_parent,
-                             const ScrubHeaderRefConst& header,
+                             ScrubHeaderRef& header,
                              MDSInternalContextBase *f)
 {
   dout(20) << __func__ << " with scrub_version " << get_version() << dendl;

diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index 2ced5b22b357190e35d8deb1906a5a98bafa401e..364f42a516c05c816227f48ba6ad8bb9289d3aa7 100644 (file)
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -273,7 +273,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
     /// my own (temporary) stamps and versions for each dirfrag we have
     std::map<frag_t, scrub_stamp_info_t> dirfrag_stamps;
 
-    ScrubHeaderRefConst header;
+    ScrubHeaderRef header;
 
     scrub_info_t() : scrub_stamp_info_t(),
        scrub_parent(NULL), on_finish(NULL),
@@ -287,6 +287,14 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
     return scrub_infop;
   }
 
+  ScrubHeaderRef get_scrub_header() {
+    if (scrub_infop == nullptr) {
+      return nullptr;
+    } else {
+      return scrub_infop->header;
+    }
+  }
+
   bool scrub_is_in_progress() const {
     return (scrub_infop && scrub_infop->scrub_in_progress);
   }
@@ -299,7 +307,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
    * directory's get_projected_version())
    */
   void scrub_initialize(CDentry *scrub_parent,
-                       const ScrubHeaderRefConst& header,
+                       ScrubHeaderRef& header,
                        MDSInternalContextBase *f);
   /**
    * Get the next dirfrag to scrub. Gives you a frag_t in output param which

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 118aff2906248f252ae36ed76ebfd543bfea6fa5..f8e48e81bd905e603ad96a9ecf041bd404b3a4f3 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -12111,14 +12111,45 @@ void MDCache::enqueue_scrub_work(MDRequestRef& mdr)
 
   header->set_origin(in);
 
-  // only set completion context for non-recursive scrub, because we don't 
-  // want to block asok caller on long running scrub
+  Context *fin = nullptr;
+  if (!header->get_recursive()) {
+    cs->take_finisher();
+  }
+
+  // If the scrub did some repair, then flush the journal at the end of
+  // the scrub.  Otherwise in the case of e.g. rewriting a backtrace
+  // the on disk state will still look damaged.
+  auto expiry_fin = new FunctionContext([this, header, fin](int r){
+      if (header->get_repaired()) {
+        dout(4) << "Flushing journal because scrub did some repairs" << dendl;
+        mds->mdlog->start_new_segment();
+        mds->mdlog->trim_all();
+        if (fin) {
+          MDSGatherBuilder expiry_gather(g_ceph_context);
+          const std::set<LogSegment*> &expiring_segments = mds->mdlog->get_expiring_segments();
+          for (std::set<LogSegment*>::const_iterator i = expiring_segments.begin();
+               i != expiring_segments.end(); ++i) {
+            (*i)->wait_for_expiry(expiry_gather.new_sub());
+          }
+          expiry_gather.set_finisher(new MDSInternalContextWrapper(mds, fin));
+          expiry_gather.activate();
+        }
+      } else {
+        if (fin) {
+          fin->complete(r);
+        }
+      }
+  });
+
   if (!header->get_recursive()) {
-    Context *fin = cs->take_finisher();
     mds->scrubstack->enqueue_inode_top(in, header,
-                                      new MDSInternalContextWrapper(mds, fin));
-  } else
-    mds->scrubstack->enqueue_inode_bottom(in, header, NULL);
+                                      new MDSInternalContextWrapper(mds,
+                                         expiry_fin));
+  } else {
+    mds->scrubstack->enqueue_inode_bottom(in, header, 
+                                      new MDSInternalContextWrapper(mds,
+                                         expiry_fin));
+  }
 
   mds->server->respond_to_request(mdr, 0);
   return;

diff --git a/src/mds/ScrubHeader.h b/src/mds/ScrubHeader.h
index 18c68ae8f9d475f2f5d30169e509a9b788d7c025..c22683d0280814fcbefa3a1fd053da9eb04584f0 100644 (file)
--- a/src/mds/ScrubHeader.h
+++ b/src/mds/ScrubHeader.h
@@ -43,6 +43,9 @@ public:
   const std::string &get_tag() const { return tag; }
   Formatter &get_formatter() const { return *formatter; }
 
+  bool get_repaired() const { return repaired; }
+  void set_repaired() { repaired = true; }
+
 protected:
   const std::string tag;
   const bool force;
@@ -50,6 +53,8 @@ protected:
   const bool repair;
   Formatter * const formatter;
   CInode *origin;
+
+  bool repaired = false;  // May be set during scrub if repairs happened
 };
 
 typedef ceph::shared_ptr<ScrubHeader> ScrubHeaderRef;

diff --git a/src/mds/ScrubStack.cc b/src/mds/ScrubStack.cc
index 7626b4e61a12fee05d7dbeac6d71b4afb0f0b5a2..83ae53abe3336f4af234b2f35c2d327f5c553ea6 100644 (file)
--- a/src/mds/ScrubStack.cc
+++ b/src/mds/ScrubStack.cc
@@ -59,7 +59,7 @@ void ScrubStack::pop_inode(CInode *in)
 }
 
 void ScrubStack::_enqueue_inode(CInode *in, CDentry *parent,
-                               const ScrubHeaderRefConst& header,
+                               ScrubHeaderRef& header,
                                MDSInternalContextBase *on_finish, bool top)
 {
   dout(10) << __func__ << " with {" << *in << "}"
@@ -72,7 +72,7 @@ void ScrubStack::_enqueue_inode(CInode *in, CDentry *parent,
     push_inode_bottom(in);
 }
 
-void ScrubStack::enqueue_inode(CInode *in, const ScrubHeaderRefConst& header,
+void ScrubStack::enqueue_inode(CInode *in, ScrubHeaderRef& header,
                                MDSInternalContextBase *on_finish, bool top)
 {
   _enqueue_inode(in, NULL, header, on_finish, top);
@@ -134,7 +134,8 @@ void ScrubStack::scrub_dir_inode(CInode *in,
   bool all_frags_terminal = true;
   bool all_frags_done = true;
 
-  const ScrubHeaderRefConst& header = in->scrub_info()->header;
+  ScrubHeaderRef header = in->get_scrub_header();
+  assert(header != nullptr);
 
   if (header->get_recursive()) {
     list<frag_t> scrubbing_frags;
@@ -289,7 +290,7 @@ void ScrubStack::scrub_dir_inode_final(CInode *in)
 }
 
 void ScrubStack::scrub_dirfrag(CDir *dir,
-                              const ScrubHeaderRefConst& header,
+                              ScrubHeaderRef& header,
                               bool *added_children, bool *is_terminal,
                               bool *done)
 {

diff --git a/src/mds/ScrubStack.h b/src/mds/ScrubStack.h
index c35b1aa9c4632c4a97518991f272410d205242f2..848efdc841f7e17305d165a5586c0dbf4af86b41 100644 (file)
--- a/src/mds/ScrubStack.h
+++ b/src/mds/ScrubStack.h
@@ -73,14 +73,14 @@ public:
    * @param header The ScrubHeader propagated from whereever this scrub
    *               was initiated
    */
-  void enqueue_inode_top(CInode *in, const ScrubHeaderRefConst& header,
+  void enqueue_inode_top(CInode *in, ScrubHeaderRef& header,
                         MDSInternalContextBase *on_finish) {
     enqueue_inode(in, header, on_finish, true);
   }
   /** Like enqueue_inode_top, but we wait for all pending scrubs before
    * starting this one.
    */
-  void enqueue_inode_bottom(CInode *in, const ScrubHeaderRefConst& header,
+  void enqueue_inode_bottom(CInode *in, ScrubHeaderRef& header,
                            MDSInternalContextBase *on_finish) {
     enqueue_inode(in, header, on_finish, false);
   }
@@ -90,9 +90,9 @@ private:
    * Put the inode at either the top or bottom of the stack, with
    * the given scrub params, and then try and kick off more scrubbing.
    */
-  void enqueue_inode(CInode *in, const ScrubHeaderRefConst& header,
+  void enqueue_inode(CInode *in, ScrubHeaderRef& header,
                       MDSInternalContextBase *on_finish, bool top);
-  void _enqueue_inode(CInode *in, CDentry *parent, const ScrubHeaderRefConst& header,
+  void _enqueue_inode(CInode *in, CDentry *parent, ScrubHeaderRef& header,
                       MDSInternalContextBase *on_finish, bool top);
   /**
    * Kick off as many scrubs as are appropriate, based on the current
@@ -164,7 +164,7 @@ private:
    * progress. Try again later.
    *
    */
-  void scrub_dirfrag(CDir *dir, const ScrubHeaderRefConst& header,
+  void scrub_dirfrag(CDir *dir, ScrubHeaderRef& header,
                     bool *added_children, bool *is_terminal, bool *done);
   /**
    * Scrub a directory-representing dentry.