mds: setup readonly mode for PurgeQueue

If the PQ faces an error, it should go read-only along with the MDS rank.

Fixes: http://tracker.ceph.com/issues/37543
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/src/mds/PurgeQueue.cc b/src/mds/PurgeQueue.cc
index 62c77184da2cc67a13e9d1e65bc3400bbf25cb99..a93c9d7218185b40746ecbd8e636b197709b7f74 100644 (file)
--- a/src/mds/PurgeQueue.cc
+++ b/src/mds/PurgeQueue.cc
@@ -19,6 +19,7 @@
 
 #include "PurgeQueue.h"
 
+#include <string.h>
 
 #define dout_context cct
 #define dout_subsys ceph_subsys_mds
@@ -139,6 +140,12 @@ void PurgeQueue::init()
 void PurgeQueue::activate()
 {
   std::lock_guard l(lock);
+
+  if (readonly) {
+    dout(10) << "skipping activate: PurgeQueue is readonly" << dendl;
+    return;
+  }
+
   if (journaler.get_read_pos() == journaler.get_write_pos())
     return;
 
@@ -193,7 +200,7 @@ void PurgeQueue::open(Context *completion)
       finish_contexts(g_ceph_context, waiting_for_recovery);
     } else {
       derr << "Error " << r << " loading Journaler" << dendl;
-      on_error->complete(r);
+      _go_readonly(r);
     }
   }));
 }
@@ -201,10 +208,14 @@ void PurgeQueue::open(Context *completion)
 void PurgeQueue::wait_for_recovery(Context* c)
 {
   std::lock_guard l(lock);
-  if (recovered)
+  if (recovered) {
     c->complete(0);
-  else
+  } else if (readonly) {
+    dout(10) << "cannot wait for recovery: PurgeQueue is readonly" << dendl;
+    c->complete(-EROFS);
+  } else {
     waiting_for_recovery.push_back(c);
+  }
 }
 
 void PurgeQueue::_recover()
@@ -226,7 +237,7 @@ void PurgeQueue::_recover()
     if (journaler.get_error()) {
       int r = journaler.get_error();
       derr << "Error " << r << " recovering write_pos" << dendl;
-      on_error->complete(r);
+      _go_readonly(r);
       return;
     }
 
@@ -260,8 +271,12 @@ void PurgeQueue::create(Context *fin)
   journaler.create(&layout, JOURNAL_FORMAT_RESILIENT);
   journaler.write_head(new FunctionContext([this](int r) {
     std::lock_guard l(lock);
-    recovered = true;
-    finish_contexts(g_ceph_context, waiting_for_recovery);
+    if (r) {
+      _go_readonly(r);
+    } else {
+      recovered = true;
+      finish_contexts(g_ceph_context, waiting_for_recovery);
+    }
   }));
 }
 
@@ -273,6 +288,12 @@ void PurgeQueue::push(const PurgeItem &pi, Context *completion)
   dout(4) << "pushing inode " << pi.ino << dendl;
   std::lock_guard l(lock);
 
+  if (readonly) {
+    dout(10) << "cannot push inode: PurgeQueue is readonly" << dendl;
+    completion->complete(-EROFS);
+    return;
+  }
+
   // Callers should have waited for open() before using us
   ceph_assert(!journaler.is_readonly());
 
@@ -335,6 +356,11 @@ uint32_t PurgeQueue::_calculate_ops(const PurgeItem &item) const
 
 bool PurgeQueue::_can_consume()
 {
+  if (readonly) {
+    dout(10) << "can't consume: PurgeQueue is readonly" << dendl;
+    return false;
+  }
+
   dout(20) << ops_in_flight << "/" << max_purge_ops << " ops, "
            << in_flight.size() << "/" << g_conf()->mds_max_purge_files
            << " files" << dendl;
@@ -362,6 +388,17 @@ bool PurgeQueue::_can_consume()
   }
 }
 
+void PurgeQueue::_go_readonly(int r)
+{
+  if (readonly) return;
+  dout(1) << "going readonly because internal IO failed: " << strerror(-r) << dendl;
+  readonly = true;
+  on_error->complete(r);
+  on_error = nullptr;
+  journaler.set_readonly();
+  finish_contexts(g_ceph_context, waiting_for_recovery, r);
+}
+
 bool PurgeQueue::_consume()
 {
   ceph_assert(lock.is_locked_by_me());
@@ -379,7 +416,7 @@ bool PurgeQueue::_consume()
 
     if (int r = journaler.get_error()) {
       derr << "Error " << r << " recovering write_pos" << dendl;
-      on_error->complete(r);
+      _go_readonly(r);
       return could_consume;
     }
 
@@ -393,7 +430,7 @@ bool PurgeQueue::_consume()
           if (r == 0) {
             _consume();
           } else if (r != -EAGAIN) {
-            on_error->complete(r);
+            _go_readonly(r);
           }
         }));
       }
@@ -415,7 +452,7 @@ bool PurgeQueue::_consume()
     } catch (const buffer::error &err) {
       derr << "Decode error at read_pos=0x" << std::hex
            << journaler.get_read_pos() << dendl;
-      on_error->complete(0);
+      _go_readonly(EIO);
     }
     dout(20) << " executing item (" << item.ino << ")" << dendl;
     _execute_item(item, journaler.get_read_pos());
@@ -582,6 +619,11 @@ void PurgeQueue::update_op_limit(const MDSMap &mds_map)
 {
   std::lock_guard l(lock);
 
+  if (readonly) {
+    dout(10) << "skipping; PurgeQueue is readonly" << dendl;
+    return;
+  }
+
   uint64_t pg_count = 0;
   objecter->with_osdmap([&](const OSDMap& o) {
     // Number of PGs across all data pools
@@ -639,6 +681,11 @@ bool PurgeQueue::drain(
 {
   std::lock_guard l(lock);
 
+  if (readonly) {
+    dout(10) << "skipping drain; PurgeQueue is readonly" << dendl;
+    return true;
+  }
+
   ceph_assert(progress != nullptr);
   ceph_assert(progress_total != nullptr);
   ceph_assert(in_flight_count != nullptr);

diff --git a/src/mds/PurgeQueue.h b/src/mds/PurgeQueue.h
index f762a426f769a72d3c9716f709735f99e8db7536..feb304019bd50bfe70c38060e070bc3521a3c771 100644 (file)
--- a/src/mds/PurgeQueue.h
+++ b/src/mds/PurgeQueue.h
@@ -106,6 +106,7 @@ protected:
   CephContext *cct;
   const mds_rank_t rank;
   Mutex lock;
+  bool readonly = false;
 
   int64_t metadata_pool;
 
@@ -164,6 +165,8 @@ protected:
   bool recovered;
   std::list<Context*> waiting_for_recovery;
 
+  void _go_readonly(int r);
+
 public:
   void init();
   void activate();