mds: setup readonly mode for PurgeQueue

If the PQ faces an error, it should go read-only along with the MDS rank.

Fixes: http://tracker.ceph.com/issues/37543
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 4cccc4dffb0915ef9e7d3b446e9a32f277646562)

Conflicts:
	src/mds/PurgeQueue.cc
- Mutex::Locker instead of std::lock_guard
- no ceph_assert in mimic

diff --git a/src/mds/PurgeQueue.cc b/src/mds/PurgeQueue.cc
index cb8de5e6ddb0f8ae38be75f16d5df153ed9b17bd..8f122e6f35284137dc3a21533d6f68d70c364c90 100644 (file)
--- a/src/mds/PurgeQueue.cc
+++ b/src/mds/PurgeQueue.cc
@@ -19,6 +19,7 @@
 
 #include "PurgeQueue.h"
 
+#include <string.h>
 
 #define dout_context cct
 #define dout_subsys ceph_subsys_mds
@@ -163,6 +164,12 @@ void PurgeQueue::init()
 void PurgeQueue::activate()
 {
   Mutex::Locker l(lock);
+
+  if (readonly) {
+    dout(10) << "skipping activate: PurgeQueue is readonly" << dendl;
+    return;
+  }
+
   if (journaler.get_read_pos() == journaler.get_write_pos())
     return;
 
@@ -217,7 +224,7 @@ void PurgeQueue::open(Context *completion)
       finish_contexts(g_ceph_context, waiting_for_recovery);
     } else {
       derr << "Error " << r << " loading Journaler" << dendl;
-      on_error->complete(r);
+      _go_readonly(r);
     }
   }));
 }
@@ -225,10 +232,14 @@ void PurgeQueue::open(Context *completion)
 void PurgeQueue::wait_for_recovery(Context* c)
 {
   Mutex::Locker l(lock);
-  if (recovered)
+  if (recovered) {
     c->complete(0);
-  else
+  } else if (readonly) {
+    dout(10) << "cannot wait for recovery: PurgeQueue is readonly" << dendl;
+    c->complete(-EROFS);
+  } else {
     waiting_for_recovery.push_back(c);
+  }
 }
 
 void PurgeQueue::_recover()
@@ -250,7 +261,7 @@ void PurgeQueue::_recover()
     if (journaler.get_error()) {
       int r = journaler.get_error();
       derr << "Error " << r << " recovering write_pos" << dendl;
-      on_error->complete(r);
+      _go_readonly(r);
       return;
     }
 
@@ -284,8 +295,12 @@ void PurgeQueue::create(Context *fin)
   journaler.create(&layout, JOURNAL_FORMAT_RESILIENT);
   journaler.write_head(new FunctionContext([this](int r) {
     Mutex::Locker l(lock);
-    recovered = true;
-    finish_contexts(g_ceph_context, waiting_for_recovery);
+    if (r) {
+      _go_readonly(r);
+    } else {
+      recovered = true;
+      finish_contexts(g_ceph_context, waiting_for_recovery);
+    }
   }));
 }
 
@@ -297,6 +312,12 @@ void PurgeQueue::push(const PurgeItem &pi, Context *completion)
   dout(4) << "pushing inode " << pi.ino << dendl;
   Mutex::Locker l(lock);
 
+  if (readonly) {
+    dout(10) << "cannot push inode: PurgeQueue is readonly" << dendl;
+    completion->complete(-EROFS);
+    return;
+  }
+
   // Callers should have waited for open() before using us
   assert(!journaler.is_readonly());
 
@@ -359,6 +380,11 @@ uint32_t PurgeQueue::_calculate_ops(const PurgeItem &item) const
 
 bool PurgeQueue::_can_consume()
 {
+  if (readonly) {
+    dout(10) << "can't consume: PurgeQueue is readonly" << dendl;
+    return false;
+  }
+
   dout(20) << ops_in_flight << "/" << max_purge_ops << " ops, "
            << in_flight.size() << "/" << g_conf->mds_max_purge_files
            << " files" << dendl;
@@ -386,6 +412,17 @@ bool PurgeQueue::_can_consume()
   }
 }
 
+void PurgeQueue::_go_readonly(int r)
+{
+  if (readonly) return;
+  dout(1) << "going readonly because internal IO failed: " << strerror(-r) << dendl;
+  readonly = true;
+  on_error->complete(r);
+  on_error = nullptr;
+  journaler.set_readonly();
+  finish_contexts(g_ceph_context, waiting_for_recovery, r);
+}
+
 bool PurgeQueue::_consume()
 {
   assert(lock.is_locked_by_me());
@@ -403,7 +440,7 @@ bool PurgeQueue::_consume()
 
     if (int r = journaler.get_error()) {
       derr << "Error " << r << " recovering write_pos" << dendl;
-      on_error->complete(r);
+      _go_readonly(r);
       return could_consume;
     }
 
@@ -417,7 +454,7 @@ bool PurgeQueue::_consume()
           if (r == 0) {
             _consume();
           } else if (r != -EAGAIN) {
-            on_error->complete(r);
+            _go_readonly(r);
           }
         }));
       }
@@ -439,7 +476,7 @@ bool PurgeQueue::_consume()
     } catch (const buffer::error &err) {
       derr << "Decode error at read_pos=0x" << std::hex
            << journaler.get_read_pos() << dendl;
-      on_error->complete(0);
+      _go_readonly(EIO);
     }
     dout(20) << " executing item (" << item.ino << ")" << dendl;
     _execute_item(item, journaler.get_read_pos());
@@ -606,6 +643,11 @@ void PurgeQueue::update_op_limit(const MDSMap &mds_map)
 {
   Mutex::Locker l(lock);
 
+  if (readonly) {
+    dout(10) << "skipping; PurgeQueue is readonly" << dendl;
+    return;
+  }
+
   uint64_t pg_count = 0;
   objecter->with_osdmap([&](const OSDMap& o) {
     // Number of PGs across all data pools
@@ -664,6 +706,11 @@ bool PurgeQueue::drain(
 {
   Mutex::Locker l(lock);
 
+  if (readonly) {
+    dout(10) << "skipping drain; PurgeQueue is readonly" << dendl;
+    return true;
+  }
+
   assert(progress != nullptr);
   assert(progress_total != nullptr);
   assert(in_flight_count != nullptr);

diff --git a/src/mds/PurgeQueue.h b/src/mds/PurgeQueue.h
index 92c677260a063ce88c8aec728e8e4cdabb1da0da..c18ea794acb9bca35d43e4507ea783eb94cee52f 100644 (file)
--- a/src/mds/PurgeQueue.h
+++ b/src/mds/PurgeQueue.h
@@ -106,6 +106,7 @@ protected:
   CephContext *cct;
   const mds_rank_t rank;
   Mutex lock;
+  bool readonly = false;
 
   int64_t metadata_pool;
 
@@ -164,6 +165,8 @@ protected:
   bool recovered;
   std::list<Context*> waiting_for_recovery;
 
+  void _go_readonly(int r);
+
 public:
   void init();
   void activate();