qa: add test support for the alloc ino failing

Fixes: https://tracker.ceph.com/issues/52280
Signed-off-by: Xiubo Li <xiubli@redhat.com>
(cherry picked from commit 71797091a25d3153d12def815ee7bd9b361593cd)

 Conflicts:
	doc/cephfs/mds-config-ref.rst: format changed
	src/common/options/mds.yaml.in: no such file

diff --git a/doc/cephfs/mds-config-ref.rst b/doc/cephfs/mds-config-ref.rst
index 2efc83b410eab7367b4cb43c4feec8662d406eff..2b22a844f3aaca31151ea0eefafc74d5248910da 100644 (file)
--- a/doc/cephfs/mds-config-ref.rst
+++ b/doc/cephfs/mds-config-ref.rst
@@ -501,6 +501,25 @@
 :Type:  32-bit Integer
 :Default: ``0``
 
+``mds_inject_skip_replaying_inotable``
+
+:Description: Ceph will skip replaying the inotable when replaying the journal,
+              and the premary MDS will crash, while the replacing MDS won't.
+              (for developers only).
+
+:Type:  Boolean
+:Default: ``false``
+
+
+``mds_kill_skip_replaying_inotable``
+
+:Description: Ceph will skip replaying the inotable when replaying the journal,
+              and the premary MDS will crash, while the replacing MDS won't.
+              (for developers only).
+
+:Type:  Boolean
+:Default: ``false``
+
 
 ``mds_wipe_sessions``
 

diff --git a/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml b/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml
index 40d63ba792b11ea5110d73e8de0cff99488bc55d..e6d6ef99b15d6ac2a6cdc1affe948fd7685ee97f 100644 (file)
--- a/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml
+++ b/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml
@@ -11,3 +11,4 @@ overrides:
       - has not responded to cap revoke by MDS for over
       - MDS_CLIENT_LATE_RELEASE
       - responding to mclientcaps
+      - RECENT_CRASH

diff --git a/qa/tasks/cephfs/test_misc.py b/qa/tasks/cephfs/test_misc.py
index 0bd8ad6217ec1e702994cf32057c426c7477bc31..4fde5fb8f5e50ce3e2390804e18dedd76851cc28 100644 (file)
--- a/qa/tasks/cephfs/test_misc.py
+++ b/qa/tasks/cephfs/test_misc.py
@@ -414,3 +414,45 @@ class TestCacheDrop(CephFSTestCase):
         # particular operation causing this is journal flush which causes the
         # MDS to wait wait for cap revoke.
         self.mount_a.resume_netns()
+
+class TestSkipReplayInoTable(CephFSTestCase):
+    MDSS_REQUIRED = 1
+    CLIENTS_REQUIRED = 1
+
+    def test_alloc_cinode_assert(self):
+        """
+        Test alloc CInode assert.
+
+        See: https://tracker.ceph.com/issues/52280
+        """
+
+        # Create a directory and the mds will journal this and then crash
+        self.mount_a.run_shell(["rm", "-rf", "test_alloc_ino"])
+        self.mount_a.run_shell(["mkdir", "test_alloc_ino"])
+
+        status = self.fs.status()
+        rank0 = self.fs.get_rank(rank=0, status=status)
+
+        self.fs.mds_asok(['config', 'set', 'mds_kill_skip_replaying_inotable', "true"])
+        # This will make the MDS crash, since we only have one MDS in the
+        # cluster and without the "wait=False" it will stuck here forever.
+        self.mount_a.run_shell(["mkdir", "test_alloc_ino/dir1"], wait=False)
+        self.fs.mds_asok(['flush', 'journal'])
+
+        # Now set the mds config to skip replaying the inotable
+        self.fs.set_ceph_conf('mds', 'mds_inject_skip_replaying_inotable', True)
+        self.fs.set_ceph_conf('mds', 'mds_wipe_sessions', True)
+
+        # sleep 5 seconds to make sure the journal log is flushed and applied
+        time.sleep(5)
+        self.fs.mds_restart()
+        # sleep 5 seconds to make sure the mds tell command won't stuck
+        time.sleep(5)
+        self.fs.wait_for_daemons()
+
+        self.delete_mds_coredump(rank0['name']);
+
+        self.mount_a.run_shell(["mkdir", "test_alloc_ino/dir2"])
+
+        ls_out = set(self.mount_a.ls("test_alloc_ino/"))
+        self.assertEqual(ls_out, set({"dir1", "dir2"}))

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index ea103d7de9ebb047527a73422ccefe5c44b1ff66..03b0973db093ec1154685dae515de99c5cb98ecf 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -130,6 +130,8 @@ OPTION(ms_connection_idle_timeout, OPT_U64)
 OPTION(ms_pq_max_tokens_per_priority, OPT_U64)
 OPTION(ms_pq_min_cost, OPT_U64)
 OPTION(ms_inject_socket_failures, OPT_U64)
+OPTION(mds_inject_skip_replaying_inotable, OPT_BOOL)
+OPTION(mds_kill_skip_replaying_inotable, OPT_BOOL)
 SAFE_OPTION(ms_inject_delay_type, OPT_STR)          // "osd mds mon client" allowed
 OPTION(ms_inject_delay_max, OPT_DOUBLE)         // seconds
 OPTION(ms_inject_delay_probability, OPT_DOUBLE) // range [0, 1]

diff --git a/src/common/options.cc b/src/common/options.cc
index 4dc69cb0db3abff1b06124e50a57fad4becf8e64..3f4527795f88e4e97399e100ee6844e77cd85bcd 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -8775,6 +8775,14 @@ std::vector<Option> get_mds_options() {
     .set_default(false)
     .set_description(""),
 
+    Option("mds_kill_skip_replaying_inotable", Option::TYPE_BOOL, Option::LEVEL_DEV)
+    .set_default(false)
+    .set_description("Ceph will skip replaying the inotable when replaying the journal, and the premary MDS will crash, while the replacing MDS won't. (for testing only)"),
+
+    Option("mds_inject_skip_replaying_inotable", Option::TYPE_BOOL, Option::LEVEL_DEV)
+    .set_default(false)
+    .set_description("Ceph will skip replaying the inotable when replaying the journal, and the premary MDS will crash, while the replacing MDS won't. (for testing only)"),
+
     Option("mds_inject_traceless_reply_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
     .set_default(0)
     .set_description(""),

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 3257067340f0c8f39c6a355ce0eee07a5286fbc9..2adbecf6ac82235c7229e2923aa29e7dac30bdbc 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -202,6 +202,9 @@ class MDCache {
   bool test_and_clear_taken_inos(inodeno_t ino) {
     return replay_taken_inos.erase(ino) != 0;
   }
+  bool is_taken_inos_empty(void) {
+    return replay_taken_inos.empty();
+  }
 
   uint64_t cache_limit_memory(void) {
     return cache_memory_limit;

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index edaf5fb3e7d1967995f87e361acfe448e6c3f8c4..314ca9b0c26c1a5b1888b2151f42c7552af5afc0 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -4435,6 +4435,9 @@ public:
   void finish(int r) override {
     ceph_assert(r == 0);
 
+    // crash current MDS and the replacing MDS will test the journal
+    ceph_assert(!g_conf()->mds_kill_skip_replaying_inotable);
+
     dn->pop_projected_linkage();
 
     // dirty inode, dn, dir
@@ -6707,6 +6710,9 @@ public:
   void finish(int r) override {
     ceph_assert(r == 0);
 
+    // crash current MDS and the replacing MDS will test the journal
+    ceph_assert(!g_conf()->mds_kill_skip_replaying_inotable);
+
     // link the inode
     dn->pop_projected_linkage();
     
@@ -7013,6 +7019,11 @@ void Server::handle_client_symlink(MDRequestRef& mdr)
 
   journal_and_reply(mdr, newi, dn, le, new C_MDS_mknod_finish(this, mdr, dn, newi));
   mds->balancer->maybe_fragment(dir, false);
+
+  // flush the journal as soon as possible
+  if (g_conf()->mds_kill_skip_replaying_inotable) {
+    mdlog->flush();
+  }
 }
 
 

diff --git a/src/mds/events/EMetaBlob.h b/src/mds/events/EMetaBlob.h
index 526ace23cb597c203b43cedc7c7321a755c3819f..357dab970579d91b03334ade15e2a080bd6ec502 100644 (file)
--- a/src/mds/events/EMetaBlob.h
+++ b/src/mds/events/EMetaBlob.h
@@ -600,7 +600,7 @@ private:
   }
 
   void update_segment(LogSegment *ls);
-  void replay(MDSRank *mds, LogSegment *ls, MDPeerUpdate *su=NULL);
+  void replay(MDSRank *mds, LogSegment *ls, int type, MDPeerUpdate *su=NULL);
 };
 WRITE_CLASS_ENCODER_FEATURES(EMetaBlob)
 WRITE_CLASS_ENCODER_FEATURES(EMetaBlob::fullbit)

diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index 74e86ec79bac993b248bff9110f0f240bc60276a..d769dd0ca1a549eba10089cb40ae025b8c25065b 100644 (file)
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -1156,7 +1156,7 @@ void EMetaBlob::generate_test_instances(std::list<EMetaBlob*>& ls)
   ls.push_back(new EMetaBlob());
 }
 
-void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDPeerUpdate *peerup)
+void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, int type, MDPeerUpdate *peerup)
 {
   dout(10) << "EMetaBlob.replay " << lump_map.size() << " dirlumps by " << client_name << dendl;
 
@@ -1560,9 +1560,12 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDPeerUpdate *peerup)
     logseg->open_files.push_back(&in->item_open_file);
   }
 
+  bool skip_replaying_inotable = g_conf()->mds_inject_skip_replaying_inotable;
+
   // allocated_inos
   if (inotablev) {
-    if (mds->inotable->get_version() >= inotablev) {
+    if (mds->inotable->get_version() >= inotablev ||
+       unlikely(type == EVENT_UPDATE && skip_replaying_inotable)) {
       dout(10) << "EMetaBlob.replay inotable tablev " << inotablev
               << " <= table " << mds->inotable->get_version() << dendl;
       if (allocated_ino)
@@ -1590,7 +1593,8 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDPeerUpdate *peerup)
     }
   }
   if (sessionmapv) {
-    if (mds->sessionmap.get_version() >= sessionmapv) {
+    if (mds->sessionmap.get_version() >= sessionmapv ||
+       unlikely(type == EVENT_UPDATE && skip_replaying_inotable)) {
       dout(10) << "EMetaBlob.replay sessionmap v " << sessionmapv
               << " <= table " << mds->sessionmap.get_version() << dendl;
       if (used_preallocated_ino)
@@ -2234,7 +2238,8 @@ void EUpdate::update_segment()
 void EUpdate::replay(MDSRank *mds)
 {
   auto&& segment = get_segment();
-  metablob.replay(mds, segment);
+  dout(10) << "EUpdate::replay" << dendl;
+  metablob.replay(mds, segment, EVENT_UPDATE);
   
   if (had_peers) {
     dout(10) << "EUpdate.replay " << reqid << " had peers, expecting a matching ECommitted" << dendl;
@@ -2317,7 +2322,7 @@ void EOpen::replay(MDSRank *mds)
 {
   dout(10) << "EOpen.replay " << dendl;
   auto&& segment = get_segment();
-  metablob.replay(mds, segment);
+  metablob.replay(mds, segment, EVENT_OPEN);
 
   // note which segments inodes belong to, so we don't have to start rejournaling them
   for (const auto &ino : inos) {
@@ -2633,7 +2638,7 @@ void EPeerUpdate::replay(MDSRank *mds)
     dout(10) << "EPeerUpdate.replay prepare " << reqid << " for mds." << leader
             << ": applying commit, saving rollback info" << dendl;
     su = new MDPeerUpdate(origop, rollback);
-    commit.replay(mds, segment, su);
+    commit.replay(mds, segment, EVENT_PEERUPDATE, su);
     mds->mdcache->add_uncommitted_peer(reqid, segment, leader, su);
     break;
 
@@ -2645,7 +2650,7 @@ void EPeerUpdate::replay(MDSRank *mds)
   case EPeerUpdate::OP_ROLLBACK:
     dout(10) << "EPeerUpdate.replay abort " << reqid << " for mds." << leader
             << ": applying rollback commit blob" << dendl;
-    commit.replay(mds, segment);
+    commit.replay(mds, segment, EVENT_PEERUPDATE);
     mds->mdcache->finish_uncommitted_peer(reqid, false);
     break;
 
@@ -2824,7 +2829,7 @@ void ESubtreeMap::replay(MDSRank *mds)
   
   // first, stick the spanning tree in my cache
   //metablob.print(*_dout);
-  metablob.replay(mds, get_segment());
+  metablob.replay(mds, get_segment(), EVENT_SUBTREEMAP);
   
   // restore import/export maps
   for (map<dirfrag_t, vector<dirfrag_t> >::iterator p = subtrees.begin();
@@ -2899,7 +2904,7 @@ void EFragment::replay(MDSRank *mds)
     ceph_abort();
   }
 
-  metablob.replay(mds, segment);
+  metablob.replay(mds, segment, EVENT_FRAGMENT);
   if (in && g_conf()->mds_debug_frag)
     in->verify_dirfrags();
 }
@@ -2983,7 +2988,7 @@ void EExport::replay(MDSRank *mds)
 {
   dout(10) << "EExport.replay " << base << dendl;
   auto&& segment = get_segment();
-  metablob.replay(mds, segment);
+  metablob.replay(mds, segment, EVENT_EXPORT);
   
   CDir *dir = mds->mdcache->get_dirfrag(base);
   ceph_assert(dir);
@@ -3062,7 +3067,7 @@ void EImportStart::replay(MDSRank *mds)
   dout(10) << "EImportStart.replay " << base << " bounds " << bounds << dendl;
   //metablob.print(*_dout);
   auto&& segment = get_segment();
-  metablob.replay(mds, segment);
+  metablob.replay(mds, segment, EVENT_IMPORTSTART);
 
   // put in ambiguous import list
   mds->mdcache->add_ambiguous_import(base, bounds);