From fd2ef7b889cfefc58eba677ac16026f6cb0cb145 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Tue, 18 Dec 2018 15:11:02 -0800 Subject: [PATCH] mds: setup readonly mode for PurgeQueue If the PQ faces an error, it should go read-only along with the MDS rank. Fixes: http://tracker.ceph.com/issues/37543 Signed-off-by: Patrick Donnelly (cherry picked from commit 4cccc4dffb0915ef9e7d3b446e9a32f277646562) Conflicts: src/mds/PurgeQueue.cc - Mutex::Locker instead of std::lock_guard - no ceph_assert in mimic --- src/mds/PurgeQueue.cc | 65 +++++++++++++++++++++++++++++++++++++------ src/mds/PurgeQueue.h | 3 ++ 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/src/mds/PurgeQueue.cc b/src/mds/PurgeQueue.cc index cb8de5e6ddb..8f122e6f352 100644 --- a/src/mds/PurgeQueue.cc +++ b/src/mds/PurgeQueue.cc @@ -19,6 +19,7 @@ #include "PurgeQueue.h" +#include #define dout_context cct #define dout_subsys ceph_subsys_mds @@ -163,6 +164,12 @@ void PurgeQueue::init() void PurgeQueue::activate() { Mutex::Locker l(lock); + + if (readonly) { + dout(10) << "skipping activate: PurgeQueue is readonly" << dendl; + return; + } + if (journaler.get_read_pos() == journaler.get_write_pos()) return; @@ -217,7 +224,7 @@ void PurgeQueue::open(Context *completion) finish_contexts(g_ceph_context, waiting_for_recovery); } else { derr << "Error " << r << " loading Journaler" << dendl; - on_error->complete(r); + _go_readonly(r); } })); } @@ -225,10 +232,14 @@ void PurgeQueue::open(Context *completion) void PurgeQueue::wait_for_recovery(Context* c) { Mutex::Locker l(lock); - if (recovered) + if (recovered) { c->complete(0); - else + } else if (readonly) { + dout(10) << "cannot wait for recovery: PurgeQueue is readonly" << dendl; + c->complete(-EROFS); + } else { waiting_for_recovery.push_back(c); + } } void PurgeQueue::_recover() @@ -250,7 +261,7 @@ void PurgeQueue::_recover() if (journaler.get_error()) { int r = journaler.get_error(); derr << "Error " << r << " recovering write_pos" << dendl; - on_error->complete(r); + _go_readonly(r); return; } @@ -284,8 +295,12 @@ void PurgeQueue::create(Context *fin) journaler.create(&layout, JOURNAL_FORMAT_RESILIENT); journaler.write_head(new FunctionContext([this](int r) { Mutex::Locker l(lock); - recovered = true; - finish_contexts(g_ceph_context, waiting_for_recovery); + if (r) { + _go_readonly(r); + } else { + recovered = true; + finish_contexts(g_ceph_context, waiting_for_recovery); + } })); } @@ -297,6 +312,12 @@ void PurgeQueue::push(const PurgeItem &pi, Context *completion) dout(4) << "pushing inode " << pi.ino << dendl; Mutex::Locker l(lock); + if (readonly) { + dout(10) << "cannot push inode: PurgeQueue is readonly" << dendl; + completion->complete(-EROFS); + return; + } + // Callers should have waited for open() before using us assert(!journaler.is_readonly()); @@ -359,6 +380,11 @@ uint32_t PurgeQueue::_calculate_ops(const PurgeItem &item) const bool PurgeQueue::_can_consume() { + if (readonly) { + dout(10) << "can't consume: PurgeQueue is readonly" << dendl; + return false; + } + dout(20) << ops_in_flight << "/" << max_purge_ops << " ops, " << in_flight.size() << "/" << g_conf->mds_max_purge_files << " files" << dendl; @@ -386,6 +412,17 @@ bool PurgeQueue::_can_consume() } } +void PurgeQueue::_go_readonly(int r) +{ + if (readonly) return; + dout(1) << "going readonly because internal IO failed: " << strerror(-r) << dendl; + readonly = true; + on_error->complete(r); + on_error = nullptr; + journaler.set_readonly(); + finish_contexts(g_ceph_context, waiting_for_recovery, r); +} + bool PurgeQueue::_consume() { assert(lock.is_locked_by_me()); @@ -403,7 +440,7 @@ bool PurgeQueue::_consume() if (int r = journaler.get_error()) { derr << "Error " << r << " recovering write_pos" << dendl; - on_error->complete(r); + _go_readonly(r); return could_consume; } @@ -417,7 +454,7 @@ bool PurgeQueue::_consume() if (r == 0) { _consume(); } else if (r != -EAGAIN) { - on_error->complete(r); + _go_readonly(r); } })); } @@ -439,7 +476,7 @@ bool PurgeQueue::_consume() } catch (const buffer::error &err) { derr << "Decode error at read_pos=0x" << std::hex << journaler.get_read_pos() << dendl; - on_error->complete(0); + _go_readonly(EIO); } dout(20) << " executing item (" << item.ino << ")" << dendl; _execute_item(item, journaler.get_read_pos()); @@ -606,6 +643,11 @@ void PurgeQueue::update_op_limit(const MDSMap &mds_map) { Mutex::Locker l(lock); + if (readonly) { + dout(10) << "skipping; PurgeQueue is readonly" << dendl; + return; + } + uint64_t pg_count = 0; objecter->with_osdmap([&](const OSDMap& o) { // Number of PGs across all data pools @@ -664,6 +706,11 @@ bool PurgeQueue::drain( { Mutex::Locker l(lock); + if (readonly) { + dout(10) << "skipping drain; PurgeQueue is readonly" << dendl; + return true; + } + assert(progress != nullptr); assert(progress_total != nullptr); assert(in_flight_count != nullptr); diff --git a/src/mds/PurgeQueue.h b/src/mds/PurgeQueue.h index 92c677260a0..c18ea794acb 100644 --- a/src/mds/PurgeQueue.h +++ b/src/mds/PurgeQueue.h @@ -106,6 +106,7 @@ protected: CephContext *cct; const mds_rank_t rank; Mutex lock; + bool readonly = false; int64_t metadata_pool; @@ -164,6 +165,8 @@ protected: bool recovered; std::list waiting_for_recovery; + void _go_readonly(int r); + public: void init(); void activate(); -- 2.47.3