mds: fix journal reformat failure in standbyreplay

In the 0.82 release, standbyreplay MDS daemons would try
to reformat the jouranl if they saw an older version on
disk, where this should have only been done by the active
MDS for the rank.  Depending on timing, this could cause
fatal corruption of the journal.

This change handles the following cases:
* only do reformat if not in standbyreplay (else raise EAGAIN
to keep trying til an active mds reformats it)
* if journal header goes away while in standbyreplay then raise
EAGAIN (handle rewrite happening in background)
* if journal version is greater than the max supported, suicide

Fixes: #8811
Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/src/mds/MDLog.cc b/src/mds/MDLog.cc
index 194f5001f4049ce2280ef81e013d6c9e6b5159b9..02e2ec970bfa2791a25e8ee6c40623a5022849d5 100644 (file)
--- a/src/mds/MDLog.cc
+++ b/src/mds/MDLog.cc
@@ -683,6 +683,14 @@ void MDLog::_recovery_thread(Context *completion)
   // rewrite failed part way through.  Erase the back journal
   // to clean up.
   if (jp.back) {
+    if (mds->is_standby_replay()) {
+      dout(1) << "Journal " << jp.front << " is being rewritten, "
+        << "cannot replay in standby until an active MDS completes rewrite" << dendl;
+      mds->mds_lock.Lock();
+      completion->complete(-EAGAIN);
+      mds->mds_lock.Unlock();
+      return;
+    }
     dout(1) << "Erasing journal " << jp.back << dendl;
     C_SaferCond erase_waiter;
     Journaler back(jp.back, mds->mdsmap->get_metadata_pool(), CEPH_FS_ONDISK_MAGIC,
@@ -735,7 +743,13 @@ void MDLog::_recovery_thread(Context *completion)
   }
 
   /* Check whether the front journal format is acceptable or needs re-write */
-  if (front_journal->get_stream_format() >= g_conf->mds_journal_format) {
+  if (front_journal->get_stream_format() > JOURNAL_FORMAT_MAX) {
+    dout(0) << "Journal " << jp.front << " is in unknown format " << front_journal->get_stream_format()
+            << ", does this MDS daemon require upgrade?" << dendl;
+    mds->mds_lock.Lock();
+    completion->complete(-EINVAL);
+    mds->mds_lock.Unlock();
+  } else if (front_journal->get_stream_format() >= g_conf->mds_journal_format) {
     /* Great, the journal is of current format and ready to rock, hook
      * it into this->journaler and complete */
     journaler = front_journal;
@@ -744,12 +758,22 @@ void MDLog::_recovery_thread(Context *completion)
     completion->complete(0);
     mds->mds_lock.Unlock();
   } else {
-    /* Hand off to reformat routine, which will ultimately set the
-     * completion when it has done its thing */
-    dout(1) << "Journal " << jp.front << " has old format "
-      << front_journal->get_stream_format() << ", it will now be updated" << dendl;
+    if (mds->is_standby_replay()) {
+        /* We must not try to rewrite in standby replay mode, because
+         * we do not have exclusive access to the log */
+        dout(1) << "Journal " << jp.front << " has old format, "
+          << "cannot replay in standby until an active MDS rewrites it" << dendl;
+        mds->mds_lock.Lock();
+        completion->complete(-EAGAIN);
+        mds->mds_lock.Unlock();
+    } else {
+        /* Hand off to reformat routine, which will ultimately set the
+         * completion when it has done its thing */
+        dout(1) << "Journal " << jp.front << " has old format "
+          << front_journal->get_stream_format() << ", it will now be updated" << dendl;
 
-    _reformat_journal(jp, front_journal, completion);
+        _reformat_journal(jp, front_journal, completion);
+    }
   }
 }
 
@@ -923,22 +947,23 @@ void MDLog::_replay_thread()
            * the MDS is going to either shut down or restart when
            * we return this error, doing it synchronously is fine
            * -- as long as we drop the main mds lock--. */
-          Mutex mylock("MDLog::_replay_thread lock");
-          Cond cond;
-          bool done = false;
-          int err = 0;
-          journaler->reread_head(new C_SafeCond(&mylock, &cond, &done, &err));
+          C_SaferCond reread_fin;
+          journaler->reread_head(&reread_fin);
           mds->mds_lock.Unlock();
-         mylock.Lock();
-          while (!done)
-            cond.Wait(mylock);
-         mylock.Unlock();
-          if (err) { // well, crap
-            dout(0) << "got error while reading head: " << cpp_strerror(err)
-                    << dendl;
-            mds->suicide();
-          }
+          int err = reread_fin.wait();
           mds->mds_lock.Lock();
+          if (err) {
+            if (err == -ENOENT && mds->is_standby_replay()) {
+              r = -EAGAIN;
+              dout(1) << "Journal header went away while in standby replay, journal rewritten?"
+                      << dendl;
+              break;
+            } else {
+                dout(0) << "got error while reading head: " << cpp_strerror(err)
+                        << dendl;
+                mds->suicide();
+            }
+          }
          standby_trim_segments();
           if (journaler->get_read_pos() < journaler->get_expire_pos()) {
             dout(0) << "expire_pos is higher than read_pos, returning EAGAIN" << dendl;

diff --git a/src/osdc/Journaler.h b/src/osdc/Journaler.h
index cce8db31f17557f0b8803f9c495b81b1d63370b0..0a0c868f823906dde3175ddcc4655f2943c68ae2 100644 (file)
--- a/src/osdc/Journaler.h
+++ b/src/osdc/Journaler.h
@@ -70,6 +70,8 @@ typedef __u8 stream_format_t;
 #define JOURNAL_FORMAT_RESILIENT 1
 #define JOURNAL_ENVELOPE_RESILIENT (sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint64_t))
 
+// Most recent format which we may try to read
+#define JOURNAL_FORMAT_MAX 1
 
 /**
  * Represents a collection of entries serialized in a byte stream.