From 9b9e3edfac973ce87663e8ae829a3f2c2680817c Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Mon, 1 Dec 2014 10:11:06 +0800 Subject: [PATCH] mds: allow choosing action for wirte error Signed-off-by: Yan, Zheng --- src/common/config_opts.h | 2 ++ src/mds/CDir.cc | 13 ++++++++++--- src/mds/CDir.h | 2 +- src/mds/CInode.cc | 30 ++++++++++++++++++++++-------- src/mds/CInode.h | 4 ++-- src/mds/MDLog.cc | 16 +++++++++++++++- src/mds/MDLog.h | 13 ++----------- src/mds/MDS.cc | 19 ++++++++++++++++++- src/mds/MDS.h | 1 + src/mds/MDSTable.cc | 15 +++++++-------- 10 files changed, 80 insertions(+), 35 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index d34a147b4c9fb..a8be56922412a 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -408,6 +408,8 @@ OPTION(mds_snap_min_uid, OPT_U32, 0) // The minimum UID required to create a sna OPTION(mds_snap_max_uid, OPT_U32, 65536) // The maximum UID allowed to create a snapshot OPTION(mds_verify_backtrace, OPT_U32, 1) +OPTION(mds_action_on_write_error, OPT_U32, 1) // 0: ignore; 1: force readonly; 2: crash + // If true, compact leveldb store on mount OPTION(osd_compact_leveldb_on_mount, OPT_BOOL, false) diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc index ec270769dbed6..7fc2ae4156212 100644 --- a/src/mds/CDir.cc +++ b/src/mds/CDir.cc @@ -1789,8 +1789,7 @@ class C_IO_Dir_Committed : public CDirIOContext { public: C_IO_Dir_Committed(CDir *d, version_t v) : CDirIOContext(d), version(v) { } void finish(int r) { - assert(r == 0); - dir->_committed(version); + dir->_committed(r, version); } }; @@ -1994,8 +1993,16 @@ void CDir::_commit(version_t want, int op_prio) * * @param v version i just committed */ -void CDir::_committed(version_t v) +void CDir::_committed(int r, version_t v) { + if (r < 0) { + dout(1) << "commit error " << r << " v " << v << dendl; + cache->mds->clog->error() << "failed to commit dir " << dirfrag() << " object," + << " errno " << r << "\n"; + cache->mds->handle_write_error(r); + return; + } + dout(10) << "_committed v " << v << " on " << *this << dendl; assert(is_auth()); diff --git a/src/mds/CDir.h b/src/mds/CDir.h index 4617f5573d732..a4d617efe2942 100644 --- a/src/mds/CDir.h +++ b/src/mds/CDir.h @@ -497,7 +497,7 @@ protected: void _commit(version_t want, int op_prio); void _omap_commit(int op_prio); void _encode_dentry(CDentry *dn, bufferlist& bl, const std::set *snaps); - void _committed(version_t v); + void _committed(int r, version_t v); public: #if 0 // unused? void wait_for_commit(Context *c, version_t v=0); diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc index 3147ead0ca9f6..5eeda497f713b 100644 --- a/src/mds/CInode.cc +++ b/src/mds/CInode.cc @@ -904,8 +904,7 @@ struct C_IO_Inode_Stored : public CInodeIOContext { Context *fin; C_IO_Inode_Stored(CInode *i, version_t v, Context *f) : CInodeIOContext(i), version(v), fin(f) {} void finish(int r) { - assert(r == 0); - in->_stored(version, fin); + in->_stored(r, version, fin); } }; @@ -943,9 +942,17 @@ void CInode::store(MDSInternalContextBase *fin) NULL, newfin); } -void CInode::_stored(version_t v, Context *fin) +void CInode::_stored(int r, version_t v, Context *fin) { - dout(10) << "_stored " << v << " " << *this << dendl; + if (r < 0) { + dout(1) << "store error " << r << " v " << v << " on " << *this << dendl; + mdcache->mds->clog->error() << "failed to store ino " << ino() << " object," + << " errno " << r << "\n"; + mdcache->mds->handle_write_error(r); + return; + } + + dout(10) << "_stored " << v << " on " << *this << dendl; if (v == get_projected_version()) mark_clean(); @@ -1062,8 +1069,7 @@ struct C_IO_Inode_StoredBacktrace : public CInodeIOContext { Context *fin; C_IO_Inode_StoredBacktrace(CInode *i, version_t v, Context *f) : CInodeIOContext(i), version(v), fin(f) {} void finish(int r) { - assert(r == 0); - in->_stored_backtrace(version, fin); + in->_stored_backtrace(r, version, fin); } }; @@ -1130,9 +1136,17 @@ void CInode::store_backtrace(MDSInternalContextBase *fin, int op_prio) gather.activate(); } -void CInode::_stored_backtrace(version_t v, Context *fin) +void CInode::_stored_backtrace(int r, version_t v, Context *fin) { - dout(10) << "_stored_backtrace" << dendl; + if (r < 0) { + dout(1) << "store backtrace error " << r << " v " << v << dendl; + mdcache->mds->clog->error() << "failed to store backtrace on dir ino " + << ino() << " object, errno " << r << "\n"; + mdcache->mds->handle_write_error(r); + return; + } + + dout(10) << "_stored_backtrace v " << v << dendl; auth_unpin(this); if (v == inode.backtrace_version) diff --git a/src/mds/CInode.h b/src/mds/CInode.h index b2ed8cff19d48..fb27756dabe8a 100644 --- a/src/mds/CInode.h +++ b/src/mds/CInode.h @@ -544,7 +544,7 @@ public: void mark_clean(); void store(MDSInternalContextBase *fin); - void _stored(version_t cv, Context *fin); + void _stored(int r, version_t cv, Context *fin); /** * Flush a CInode to disk. This includes the backtrace, the parent * directory's link, and the Inode object itself (if a base directory). @@ -559,7 +559,7 @@ public: void build_backtrace(int64_t pool, inode_backtrace_t& bt); void store_backtrace(MDSInternalContextBase *fin, int op_prio=-1); - void _stored_backtrace(version_t v, Context *fin); + void _stored_backtrace(int r, version_t v, Context *fin); void fetch_backtrace(Context *fin, bufferlist *backtrace); void _mark_dirty_parent(LogSegment *ls, bool dirty_pool=false); void clear_dirty_parent(); diff --git a/src/mds/MDLog.cc b/src/mds/MDLog.cc index 97d105a27913e..907091b0cd947 100644 --- a/src/mds/MDLog.cc +++ b/src/mds/MDLog.cc @@ -84,7 +84,8 @@ class C_MDL_WriteError : public MDSIOContextBase { void finish(int r) { MDS *mds = get_mds(); - + // assume journal is reliable, so don't choose action based on + // g_conf->mds_action_on_write_error. if (r == -EBLACKLISTED) { derr << "we have been blacklisted (fenced), respawning..." << dendl; mds->respawn(); @@ -566,6 +567,19 @@ void MDLog::trim(int m) _trim_expired_segments(); } +class C_MaybeExpiredSegment : public MDSInternalContext { + MDLog *mdlog; + LogSegment *ls; + int op_prio; + public: + C_MaybeExpiredSegment(MDLog *mdl, LogSegment *s, int p) : + MDSInternalContext(mdl->mds), mdlog(mdl), ls(s), op_prio(p) {} + void finish(int res) { + if (res < 0) + mdlog->mds->handle_write_error(res); + mdlog->_maybe_expired(ls, op_prio); + } +}; /** * Like ::trim, but instead of trimming to max_segments, trim all but the latest diff --git a/src/mds/MDLog.h b/src/mds/MDLog.h index 601148dc912da..7b21ff258e663 100644 --- a/src/mds/MDLog.h +++ b/src/mds/MDLog.h @@ -284,22 +284,13 @@ public: } private: - class C_MaybeExpiredSegment : public MDSInternalContext { - MDLog *mdlog; - LogSegment *ls; - int op_prio; - public: - C_MaybeExpiredSegment(MDLog *mdl, LogSegment *s, int p) : MDSInternalContext(mdl->mds), mdlog(mdl), ls(s), op_prio(p) {} - void finish(int res) { - mdlog->_maybe_expired(ls, op_prio); - } - }; - void try_expire(LogSegment *ls, int op_prio); void _maybe_expired(LogSegment *ls, int op_prio); void _expired(LogSegment *ls); void _trim_expired_segments(); + friend class C_MaybeExpiredSegment; + public: void trim_expired_segments(); void trim(int max=-1); diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc index 158602fd77578..332408cf7d1ed 100644 --- a/src/mds/MDS.cc +++ b/src/mds/MDS.cc @@ -2251,8 +2251,25 @@ void MDS::respawn() suicide(); } +void MDS::handle_write_error(int err) +{ + if (err == -EBLACKLISTED) { + derr << "we have been blacklisted (fenced), respawning..." << dendl; + respawn(); + return; + } - + if (g_conf->mds_action_on_write_error >= 2) { + derr << "unhandled write error " << cpp_strerror(err) << ", suicide..." << dendl; + suicide(); + } else if (g_conf->mds_action_on_write_error == 1) { + derr << "unhandled write error " << cpp_strerror(err) << ", force readonly..." << dendl; + mdcache->force_readonly(); + } else { + // ignore; + derr << "unhandled write error " << cpp_strerror(err) << ", ignore..." << dendl; + } +} bool MDS::ms_dispatch(Message *m) { diff --git a/src/mds/MDS.h b/src/mds/MDS.h index e74f0428743a0..0ee73e37e4634 100644 --- a/src/mds/MDS.h +++ b/src/mds/MDS.h @@ -441,6 +441,7 @@ private: void suicide(); void respawn(); + void handle_write_error(int err); void tick(); diff --git a/src/mds/MDSTable.cc b/src/mds/MDSTable.cc index 36c2d0a7b110c..8e159d7c9df63 100644 --- a/src/mds/MDSTable.cc +++ b/src/mds/MDSTable.cc @@ -88,16 +88,15 @@ void MDSTable::save(MDSInternalContextBase *onfinish, version_t v) void MDSTable::save_2(int r, version_t v) { - dout(10) << "save_2 v " << v << dendl; - if (r == -EBLACKLISTED) { - mds->suicide(); - return; - } if (r < 0) { - dout(10) << "save_2 could not write table: " << r << dendl; - assert(r >= 0); + dout(1) << "save error " << r << " v " << v << dendl; + mds->clog->error() << "failed to store table " << table_name << " object," + << " errno " << r << "\n"; + mds->handle_write_error(r); + return; } - assert(r >= 0); + + dout(10) << "save_2 v " << v << dendl; committed_version = v; list ls; -- 2.39.5