mds: catch damage to dentry's first field

When possible. Abort the MDS before it can be written to the
journal/directory.

This is part of a series to address corruption first observed in [1].
How the corruption is introduced is yet unknown.

[1] https://tracker.ceph.com/issues/38452#note-10

Fixes: http://tracker.ceph.com/issues/58482
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 03407528f95fe60e5af4062c3caa3688b8b31530)

Conflicts:
	src/common/options/mds.yaml.in
	src/mds/CDir.cc

diff --git a/src/common/options.cc b/src/common/options.cc
index b6481d1f435677fdd2c6bc219e442c0860106005..d3ced762ea86bf61d8f491363417373e6cb165c4 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -8684,6 +8684,10 @@ std::vector<Option> get_mds_options() {
     .set_default(false)
     .set_description(""),
 
+    Option("mds_abort_on_newly_corrupt_dentry", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_description("MDS will abort if dentry is detected newly corrupted."),
+
     Option("mds_kill_mdstable_at", Option::TYPE_INT, Option::LEVEL_DEV)
     .set_default(0)
     .set_description(""),

diff --git a/src/mds/CDentry.cc b/src/mds/CDentry.cc
index 9337c11383c4d13411f9e5dfc56d62dff728ab9a..e6f7b5e54221d892b2ae040615e09178f5d99bf4 100644 (file)
--- a/src/mds/CDentry.cc
+++ b/src/mds/CDentry.cc
@@ -17,6 +17,7 @@
 #include "CDentry.h"
 #include "CInode.h"
 #include "CDir.h"
+#include "SnapClient.h"
 
 #include "MDSRank.h"
 #include "MDCache.h"
@@ -677,4 +678,25 @@ bool CDentry::scrub(snapid_t next_seq)
   return false;
 }
 
+bool CDentry::check_corruption(bool load)
+{
+  auto&& snapclient = dir->mdcache->mds->snapclient;
+  auto next_snap = snapclient->get_last_seq()+1;
+  if (first > last || (snapclient->is_server_ready() && first > next_snap)) {
+    if (load) {
+      dout(1) << "loaded already corrupt dentry: " << *this << dendl;
+      corrupt_first_loaded = true;
+    } else {
+      derr << "newly corrupt dentry to be committed: " << *this << dendl;
+    }
+    dir->go_bad_dentry(last, get_name());
+    if (!load && g_conf().get_val<bool>("mds_abort_on_newly_corrupt_dentry")) {
+      dir->mdcache->mds->clog->error() << "MDS abort because newly corrupt dentry to be committed: " << *this;
+      ceph_abort("detected newly corrupt dentry"); /* avoid writing out newly corrupted dn */
+    }
+    return true;
+  }
+  return false;
+}
+
 MEMPOOL_DEFINE_OBJECT_FACTORY(CDentry, co_dentry, mds_co);

diff --git a/src/mds/CDentry.h b/src/mds/CDentry.h
index 0e2a852ddd24e56c26e56e04ad2e526277920d51..03932285cf7614dcb7851424fa516c851cc3b9a8 100644 (file)
--- a/src/mds/CDentry.h
+++ b/src/mds/CDentry.h
@@ -160,6 +160,8 @@ public:
     return dentry_key_t(last, name.c_str(), hash);
   }
 
+  bool check_corruption(bool load);
+
   const CDir *get_dir() const { return dir; }
   CDir *get_dir() { return dir; }
   std::string_view get_name() const { return std::string_view(name); }
@@ -364,6 +366,7 @@ public:
 
   __u32 hash;
   snapid_t first, last;
+  bool corrupt_first_loaded = false; /* for Postgres corruption detection */
 
   elist<CDentry*>::item item_dirty, item_dir_dirty;
   elist<CDentry*>::item item_stray;

diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index b906a645efaa9515a2af1230ee74a13063d5f960..baf79d4c6c4a00524eff272c39df59b578b88b1c 100644 (file)
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -29,8 +29,10 @@
 #include "MDLog.h"
 #include "LogSegment.h"
 #include "MDBalancer.h"
+#include "SnapClient.h"
 
 #include "common/bloom_filter.hpp"
+#include "common/likely.h"
 #include "include/Context.h"
 #include "common/Clock.h"
 
@@ -347,14 +349,14 @@ CDentry* CDir::add_null_dentry(std::string_view dname,
    
   // create dentry
   CDentry* dn = new CDentry(dname, inode->hash_dentry_name(dname), "", first, last);
+  dn->dir = this;
+  dn->version = get_projected_version();
+  dn->check_corruption(true);
   if (is_auth()) 
     dn->state_set(CDentry::STATE_AUTH);
 
   mdcache->bottom_lru.lru_insert_mid(dn);
   dn->state_set(CDentry::STATE_BOTTOMLRU);
-
-  dn->dir = this;
-  dn->version = get_projected_version();
   
   // add to dir
   ceph_assert(items.count(dn->key()) == 0);
@@ -391,6 +393,9 @@ CDentry* CDir::add_primary_dentry(std::string_view dname, CInode *in,
   
   // create dentry
   CDentry* dn = new CDentry(dname, inode->hash_dentry_name(dname), std::move(alternate_name), first, last);
+  dn->dir = this;
+  dn->version = get_projected_version();
+  dn->check_corruption(true);
   if (is_auth()) 
     dn->state_set(CDentry::STATE_AUTH);
   if (is_auth() || !inode->is_stray()) {
@@ -400,9 +405,6 @@ CDentry* CDir::add_primary_dentry(std::string_view dname, CInode *in,
     dn->state_set(CDentry::STATE_BOTTOMLRU);
   }
 
-  dn->dir = this;
-  dn->version = get_projected_version();
-  
   // add to dir
   ceph_assert(items.count(dn->key()) == 0);
   //assert(null_items.count(dn->get_name()) == 0);
@@ -441,12 +443,12 @@ CDentry* CDir::add_remote_dentry(std::string_view dname, inodeno_t ino, unsigned
 
   // create dentry
   CDentry* dn = new CDentry(dname, inode->hash_dentry_name(dname), std::move(alternate_name), ino, d_type, first, last);
+  dn->dir = this;
+  dn->version = get_projected_version();
+  dn->check_corruption(true);
   if (is_auth()) 
     dn->state_set(CDentry::STATE_AUTH);
   mdcache->lru.lru_insert_mid(dn);
-
-  dn->dir = this;
-  dn->version = get_projected_version();
   
   // add to dir
   ceph_assert(items.count(dn->key()) == 0);
@@ -1756,11 +1758,6 @@ CDentry *CDir::_load_dentry(
            << " [" << first << "," << last << "]"
            << dendl;
 
-  if (first > last) {
-    go_bad_dentry(last, dname);
-    /* try to continue */
-  }
-
   bool stale = false;
   if (snaps && last != CEPH_NOSNAP) {
     set<snapid_t>::const_iterator p = snaps->lower_bound(first);
@@ -2427,6 +2424,10 @@ void CDir::_omap_commit(int op_prio)
     string key;
     dn->key().encode(key);
 
+    if (!dn->corrupt_first_loaded) {
+      dn->check_corruption(false);
+    }
+
     if (snaps && try_trim_snap_dentry(dn, *snaps)) {
       dout(10) << " rm " << key << dendl;
       to_remove.emplace_back(std::move(key));

diff --git a/src/mds/MDSTableClient.h b/src/mds/MDSTableClient.h
index 2952ec4069d20d72c285926d5796e04e553315bb..0bc88aaf29fed5d2ccc476a0211b088c9f795de1 100644 (file)
--- a/src/mds/MDSTableClient.h
+++ b/src/mds/MDSTableClient.h
@@ -58,6 +58,10 @@ public:
 
   void handle_mds_failure(mds_rank_t mds);
 
+  bool is_server_ready(void) const {
+    return server_ready;
+  }
+
   // child must implement
   virtual void resend_queries() = 0;
   virtual void handle_query_result(const cref_t<MMDSTableRequest> &m) = 0;

diff --git a/src/mds/SnapClient.h b/src/mds/SnapClient.h
index e259f7924932d0beab636dd75e4b6f4ced5287a6..b3d5ca3cb86b47f47f1d7626759c9f6de099ef78 100644 (file)
--- a/src/mds/SnapClient.h
+++ b/src/mds/SnapClient.h
@@ -87,6 +87,7 @@ public:
 
   snapid_t get_last_created() const { return cached_last_created; }
   snapid_t get_last_destroyed() const { return cached_last_destroyed; }
+  snapid_t get_last_seq() const { return std::max(cached_last_destroyed, cached_last_created); }
 
   void get_snaps(set<snapid_t>& snaps) const;
   set<snapid_t> filter(const set<snapid_t>& snaps) const;

diff --git a/src/mds/events/EMetaBlob.h b/src/mds/events/EMetaBlob.h
index ff8966b1cdbb87cfe5334ebf8f1b7ebec1a93c54..526ace23cb597c203b43cedc7c7321a755c3819f 100644 (file)
--- a/src/mds/events/EMetaBlob.h
+++ b/src/mds/events/EMetaBlob.h
@@ -417,6 +417,7 @@ private:
   }
   void add_null_dentry(dirlump& lump, CDentry *dn, bool dirty) {
     // add the dir
+    dn->check_corruption(false);
     lump.nnull++;
     lump.add_dnull(dn->get_name(), dn->first, dn->last,
                   dn->get_projected_version(), dirty);
@@ -430,6 +431,7 @@ private:
   }
   void add_remote_dentry(dirlump& lump, CDentry *dn, bool dirty, 
                         inodeno_t rino=0, unsigned char rdt=0) {
+    dn->check_corruption(false);
     if (!rino) {
       rino = dn->get_projected_linkage()->get_remote_ino();
       rdt = dn->get_projected_linkage()->get_remote_d_type();
@@ -451,6 +453,8 @@ private:
     add_primary_dentry(add_dir(dn->get_dir(), false), dn, in, state);
   }
   void add_primary_dentry(dirlump& lump, CDentry *dn, CInode *in, __u8 state) {
+    dn->check_corruption(false);
+
     if (!in) 
       in = dn->get_projected_linkage()->get_inode();