tasks: add mds_auto_repair

New CephFS tests for MDS's auto repair functions. (So far the only
test case is verify/repair backtrace on fetch dirfrag)

Signed-off-by: Yan, Zheng <zyan@redhat.com>

diff --git a/tasks/cephfs/mount.py b/tasks/cephfs/mount.py
index cf44ed7a8ab441bdd08fa457fc60542470e2edab..b977c1e233aa89c12cedcf9c0596f25f4fcc4ae0 100644 (file)
--- a/tasks/cephfs/mount.py
+++ b/tasks/cephfs/mount.py
@@ -1,4 +1,5 @@
 from contextlib import contextmanager
+from cStringIO import StringIO
 import logging
 import datetime
 import time
@@ -104,7 +105,7 @@ class CephFSMount(object):
 
     def run_shell(self, args):
         args = ["cd", self.mountpoint, run.Raw('&&')] + args
-        return self.client_remote.run(args=args)
+        return self.client_remote.run(args=args, stdout=StringIO())
 
     def open_background(self, basename="background_file"):
         """

diff --git a/tasks/mds_auto_repair.py b/tasks/mds_auto_repair.py
new file mode 100644 (file)
index 0000000..4f01d5a
--- /dev/null
+++ b/tasks/mds_auto_repair.py
@@ -0,0 +1,118 @@
+
+"""
+Exercise the MDS's auto repair functions
+"""
+
+import contextlib
+import logging
+import time
+
+from teuthology.orchestra.run import CommandFailedError
+
+from tasks.cephfs.filesystem import Filesystem
+from tasks.cephfs.fuse_mount import FuseMount
+from tasks.cephfs.cephfs_test_case import CephFSTestCase, run_tests
+
+
+log = logging.getLogger(__name__)
+
+
+# Arbitrary timeouts for operations involving restarting
+# an MDS or waiting for it to come up
+MDS_RESTART_GRACE = 60
+
+class TestMDSAutoRepair(CephFSTestCase):
+    # Environment references
+    mount_a = None
+
+    def __init__(self, *args, **kwargs):
+        super(TestMDSAutoRepair, self).__init__(*args, **kwargs)
+
+        self.configs_set = set()
+
+    def set_conf(self, subsys, key, value):
+        self.configs_set.add((subsys, key))
+        self.fs.set_ceph_conf(subsys, key, value)
+
+    def setUp(self):
+        self.fs.mds_restart()
+        self.fs.wait_for_daemons()
+        self.mount_a.mount()
+        self.mount_a.wait_until_mounted()
+
+    def tearDown(self):
+        self.fs.clear_firewall()
+        self.mount_a.teardown()
+
+        for subsys, key in self.configs_set:
+            self.fs.clear_ceph_conf(subsys, key)
+
+    def test_backtrace_repair(self):
+        """
+        MDS should verify/fix backtrace on fetch dirfrag
+        """
+
+        # trim log segment as fast as possible
+        self.fs.set_ceph_conf('mds', 'mds cache size', 100)
+        self.fs.set_ceph_conf('mds', 'mds log max segments', 2)
+        self.fs.set_ceph_conf('mds', 'mds log events per segment', 1)
+        self.fs.set_ceph_conf('mds', 'mds verify backtrace', 1)
+        self.fs.mds_restart()
+        self.fs.wait_for_daemons()
+
+        create_script = "mkdir {0}; for i in `seq 0 500`; do touch {0}/file$i; done"
+        # create main test directory
+        self.mount_a.run_shell(["sudo", "bash", "-c", create_script.format("testdir1")])
+
+        # create more files in another directory. make sure MDS trim dentries in testdir1
+        self.mount_a.run_shell(["sudo", "bash", "-c", create_script.format("testdir2")])
+
+        # flush journal entries to dirfrag objects
+        self.fs.mds_asok(['flush', 'journal'])
+
+        # drop inodes caps
+        self.mount_a.umount_wait()
+        self.mount_a.mount()
+        self.mount_a.wait_until_mounted()
+
+        # wait MDS to trim dentries in testdir1. 60 seconds should be long enough.
+        time.sleep(60)
+
+        # remove testdir1's backtrace
+        proc = self.mount_a.run_shell(["sudo", "ls", "-id", "testdir1"])
+        self.assertEqual(proc.exitstatus, 0)
+        objname = "{:x}.00000000".format(long(proc.stdout.getvalue().split()[0]))
+        proc = self.mount_a.run_shell(["sudo", "rados", "-p", "metadata", "rmxattr", objname, "parent"])
+        self.assertEqual(proc.exitstatus, 0);
+
+        # readdir (fetch dirfrag) should fix testdir1's backtrace
+        self.mount_a.run_shell(["sudo", "ls", "testdir1"])
+
+        # add more entries to journal
+        self.mount_a.run_shell(["sudo", "rm", "-rf", " testdir2"])
+
+        # flush journal entries to dirfrag objects
+        self.fs.mds_asok(['flush', 'journal'])
+
+        # check if backtrace exists
+        proc = self.mount_a.run_shell(["sudo", "rados", "-p", "metadata", "getxattr", objname, "parent"])
+        self.assertEqual(proc.exitstatus, 0)
+
+@contextlib.contextmanager
+def task(ctx, config):
+    fs = Filesystem(ctx, config)
+    mount_a = ctx.mounts.values()[0]
+
+    # Stash references on ctx so that we can easily debug in interactive mode
+    # =======================================================================
+    ctx.filesystem = fs
+    ctx.mount_a = mount_a
+
+    run_tests(ctx, config, TestMDSAutoRepair, {
+        'fs': fs,
+        'mount_a': mount_a,
+    })
+
+    # Continue to any downstream tasks
+    # ================================
+    yield