mds: allow choosing action for wirte error

Signed-off-by: Yan, Zheng <zyan@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index d34a147b4c9fb3c197439609ca8659bb02e1d47c..a8be56922412a10e2223571dbbe4159fbc761ec5 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -408,6 +408,8 @@ OPTION(mds_snap_min_uid, OPT_U32, 0) // The minimum UID required to create a sna
 OPTION(mds_snap_max_uid, OPT_U32, 65536) // The maximum UID allowed to create a snapshot
 OPTION(mds_verify_backtrace, OPT_U32, 1)
 
+OPTION(mds_action_on_write_error, OPT_U32, 1) // 0: ignore; 1: force readonly; 2: crash
+
 // If true, compact leveldb store on mount
 OPTION(osd_compact_leveldb_on_mount, OPT_BOOL, false)
 

diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index ec270769dbed6fb1b19ec0042b7143768f1cceef..7fc2ae41562121b6e420dcceb150800597a9f59c 100644 (file)
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -1789,8 +1789,7 @@ class C_IO_Dir_Committed : public CDirIOContext {
 public:
   C_IO_Dir_Committed(CDir *d, version_t v) : CDirIOContext(d), version(v) { }
   void finish(int r) {
-    assert(r == 0);
-    dir->_committed(version);
+    dir->_committed(r, version);
   }
 };
 
@@ -1994,8 +1993,16 @@ void CDir::_commit(version_t want, int op_prio)
  *
  * @param v version i just committed
  */
-void CDir::_committed(version_t v)
+void CDir::_committed(int r, version_t v)
 {
+  if (r < 0) {
+    dout(1) << "commit error " << r << " v " << v << dendl;
+    cache->mds->clog->error() << "failed to commit dir " << dirfrag() << " object,"
+                             << " errno " << r << "\n";
+    cache->mds->handle_write_error(r);
+    return;
+  }
+
   dout(10) << "_committed v " << v << " on " << *this << dendl;
   assert(is_auth());
 

diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index 4617f5573d732f7f283eb157f617b8b8d676fb61..a4d617efe294212dd17a10a5c2c841a662a544ed 100644 (file)
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -497,7 +497,7 @@ protected:
   void _commit(version_t want, int op_prio);
   void _omap_commit(int op_prio);
   void _encode_dentry(CDentry *dn, bufferlist& bl, const std::set<snapid_t> *snaps);
-  void _committed(version_t v);
+  void _committed(int r, version_t v);
 public:
 #if 0  // unused?
   void wait_for_commit(Context *c, version_t v=0);

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 3147ead0ca9f627ea0d98fbf171672e0cfd5884f..5eeda497f713b61fb3d199a4961d3fd80a0c5ce2 100644 (file)
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -904,8 +904,7 @@ struct C_IO_Inode_Stored : public CInodeIOContext {
   Context *fin;
   C_IO_Inode_Stored(CInode *i, version_t v, Context *f) : CInodeIOContext(i), version(v), fin(f) {}
   void finish(int r) {
-    assert(r == 0);
-    in->_stored(version, fin);
+    in->_stored(r, version, fin);
   }
 };
 
@@ -943,9 +942,17 @@ void CInode::store(MDSInternalContextBase *fin)
                                 NULL, newfin);
 }
 
-void CInode::_stored(version_t v, Context *fin)
+void CInode::_stored(int r, version_t v, Context *fin)
 {
-  dout(10) << "_stored " << v << " " << *this << dendl;
+  if (r < 0) {
+    dout(1) << "store error " << r << " v " << v << " on " << *this << dendl;
+    mdcache->mds->clog->error() << "failed to store ino " << ino() << " object,"
+                               << " errno " << r << "\n";
+    mdcache->mds->handle_write_error(r);
+    return;
+  }
+
+  dout(10) << "_stored " << v << " on " << *this << dendl;
   if (v == get_projected_version())
     mark_clean();
 
@@ -1062,8 +1069,7 @@ struct C_IO_Inode_StoredBacktrace : public CInodeIOContext {
   Context *fin;
   C_IO_Inode_StoredBacktrace(CInode *i, version_t v, Context *f) : CInodeIOContext(i), version(v), fin(f) {}
   void finish(int r) {
-    assert(r == 0);
-    in->_stored_backtrace(version, fin);
+    in->_stored_backtrace(r, version, fin);
   }
 };
 
@@ -1130,9 +1136,17 @@ void CInode::store_backtrace(MDSInternalContextBase *fin, int op_prio)
   gather.activate();
 }
 
-void CInode::_stored_backtrace(version_t v, Context *fin)
+void CInode::_stored_backtrace(int r, version_t v, Context *fin)
 {
-  dout(10) << "_stored_backtrace" << dendl;
+  if (r < 0) {
+    dout(1) << "store backtrace error " << r << " v " << v << dendl;
+    mdcache->mds->clog->error() << "failed to store backtrace on dir ino "
+                               << ino() << " object, errno " << r << "\n";
+    mdcache->mds->handle_write_error(r);
+    return;
+  }
+
+  dout(10) << "_stored_backtrace v " << v <<  dendl;
 
   auth_unpin(this);
   if (v == inode.backtrace_version)

diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index b2ed8cff19d489e4ad163821391e900e48882e35..fb27756dabe8a9f4e692670fbca92fa69d7fdfc0 100644 (file)
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -544,7 +544,7 @@ public:
   void mark_clean();
 
   void store(MDSInternalContextBase *fin);
-  void _stored(version_t cv, Context *fin);
+  void _stored(int r, version_t cv, Context *fin);
   /**
    * Flush a CInode to disk. This includes the backtrace, the parent
    * directory's link, and the Inode object itself (if a base directory).
@@ -559,7 +559,7 @@ public:
 
   void build_backtrace(int64_t pool, inode_backtrace_t& bt);
   void store_backtrace(MDSInternalContextBase *fin, int op_prio=-1);
-  void _stored_backtrace(version_t v, Context *fin);
+  void _stored_backtrace(int r, version_t v, Context *fin);
   void fetch_backtrace(Context *fin, bufferlist *backtrace);
   void _mark_dirty_parent(LogSegment *ls, bool dirty_pool=false);
   void clear_dirty_parent();

diff --git a/src/mds/MDLog.cc b/src/mds/MDLog.cc
index 97d105a27913e3672efcf833d283b30e9c1d9d2f..907091b0cd9471b8557af695675aebac451db831 100644 (file)
--- a/src/mds/MDLog.cc
+++ b/src/mds/MDLog.cc
@@ -84,7 +84,8 @@ class C_MDL_WriteError : public MDSIOContextBase {
 
   void finish(int r) {
     MDS *mds = get_mds();
-
+    // assume journal is reliable, so don't choose action based on
+    // g_conf->mds_action_on_write_error.
     if (r == -EBLACKLISTED) {
       derr << "we have been blacklisted (fenced), respawning..." << dendl;
       mds->respawn();
@@ -566,6 +567,19 @@ void MDLog::trim(int m)
   _trim_expired_segments();
 }
 
+class C_MaybeExpiredSegment : public MDSInternalContext {
+  MDLog *mdlog;
+  LogSegment *ls;
+  int op_prio;
+  public:
+  C_MaybeExpiredSegment(MDLog *mdl, LogSegment *s, int p) :
+    MDSInternalContext(mdl->mds), mdlog(mdl), ls(s), op_prio(p) {}
+  void finish(int res) {
+    if (res < 0)
+      mdlog->mds->handle_write_error(res);
+    mdlog->_maybe_expired(ls, op_prio);
+  }
+};
 
 /**
  * Like ::trim, but instead of trimming to max_segments, trim all but the latest

diff --git a/src/mds/MDLog.h b/src/mds/MDLog.h
index 601148dc912da68e222f8b3ce3ce42fa4fad2c17..7b21ff258e663ddb1b62b16474cb4215b88b1032 100644 (file)
--- a/src/mds/MDLog.h
+++ b/src/mds/MDLog.h
@@ -284,22 +284,13 @@ public:
   }
 
 private:
-  class C_MaybeExpiredSegment : public MDSInternalContext {
-    MDLog *mdlog;
-    LogSegment *ls;
-    int op_prio;
-  public:
-    C_MaybeExpiredSegment(MDLog *mdl, LogSegment *s, int p) : MDSInternalContext(mdl->mds), mdlog(mdl), ls(s), op_prio(p) {}
-    void finish(int res) {
-      mdlog->_maybe_expired(ls, op_prio);
-    }
-  };
-
   void try_expire(LogSegment *ls, int op_prio);
   void _maybe_expired(LogSegment *ls, int op_prio);
   void _expired(LogSegment *ls);
   void _trim_expired_segments();
 
+  friend class C_MaybeExpiredSegment;
+
 public:
   void trim_expired_segments();
   void trim(int max=-1);

diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 158602fd77578a4f1a30fd182232eb46b39e13a3..332408cf7d1ed6c896d4cd606f221a2fb37a5116 100644 (file)
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -2251,8 +2251,25 @@ void MDS::respawn()
   suicide();
 }
 
+void MDS::handle_write_error(int err)
+{
+  if (err == -EBLACKLISTED) {
+    derr << "we have been blacklisted (fenced), respawning..." << dendl;
+    respawn();
+    return;
+  }
 
-
+  if (g_conf->mds_action_on_write_error >= 2) {
+    derr << "unhandled write error " << cpp_strerror(err) << ", suicide..." << dendl;
+    suicide();
+  } else if (g_conf->mds_action_on_write_error == 1) {
+    derr << "unhandled write error " << cpp_strerror(err) << ", force readonly..." << dendl;
+    mdcache->force_readonly();
+  } else {
+    // ignore;
+    derr << "unhandled write error " << cpp_strerror(err) << ", ignore..." << dendl;
+  }
+}
 
 bool MDS::ms_dispatch(Message *m)
 {

diff --git a/src/mds/MDS.h b/src/mds/MDS.h
index e74f0428743a040c7bbc0898830b5297ce4cc1dc..0ee73e37e4634c46fec335b43d202f661b5d3ab2 100644 (file)
--- a/src/mds/MDS.h
+++ b/src/mds/MDS.h
@@ -441,6 +441,7 @@ private:
 
   void suicide();
   void respawn();
+  void handle_write_error(int err);
 
   void tick();
   

diff --git a/src/mds/MDSTable.cc b/src/mds/MDSTable.cc
index 36c2d0a7b110c66b2a4c9c8a30ee9b4035a7e485..8e159d7c9df63647c6cbc14f1d353a0cb6a5786f 100644 (file)
--- a/src/mds/MDSTable.cc
+++ b/src/mds/MDSTable.cc
@@ -88,16 +88,15 @@ void MDSTable::save(MDSInternalContextBase *onfinish, version_t v)
 
 void MDSTable::save_2(int r, version_t v)
 {
-  dout(10) << "save_2 v " << v << dendl;
-  if (r == -EBLACKLISTED) {
-    mds->suicide();
-    return;
-  }
   if (r < 0) {
-    dout(10) << "save_2 could not write table: " << r << dendl;
-    assert(r >= 0);
+    dout(1) << "save error " << r << " v " << v << dendl;
+    mds->clog->error() << "failed to store table " << table_name << " object,"
+                      << " errno " << r << "\n";
+    mds->handle_write_error(r);
+    return;
   }
-  assert(r >= 0);
+
+  dout(10) << "save_2 v " << v << dendl;
   committed_version = v;
   
   list<MDSInternalContextBase*> ls;