From: Xiubo Li Date: Wed, 23 Dec 2020 03:31:09 +0000 (+0800) Subject: qa: add run_scrub helper in filesystem class X-Git-Tag: v17.1.0~2738^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=55d7db9ed6840d7801f268d7ccf36c69952f4495;p=ceph.git qa: add run_scrub helper in filesystem class Signed-off-by: Xiubo Li --- diff --git a/qa/tasks/cephfs/cephfs_test_case.py b/qa/tasks/cephfs/cephfs_test_case.py index edcc8907ce96..0e9d23ee370e 100644 --- a/qa/tasks/cephfs/cephfs_test_case.py +++ b/qa/tasks/cephfs/cephfs_test_case.py @@ -365,7 +365,7 @@ class CephFSTestCase(CephTestCase): raise RuntimeError("rank {0} failed to reach desired subtree state".format(rank)) from e def _wait_until_scrub_complete(self, path="/", recursive=True, timeout=100): - out_json = self.fs.rank_tell(["scrub", "start", path] + ["recursive"] if recursive else []) + out_json = self.fs.run_scrub(["start", path] + ["recursive"] if recursive else []) if not self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"], sleep=10, timeout=timeout): log.info("timed out waiting for scrub to complete") diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py index 2b1942b5791b..fb407ac4506b 100644 --- a/qa/tasks/cephfs/filesystem.py +++ b/qa/tasks/cephfs/filesystem.py @@ -1566,8 +1566,11 @@ class Filesystem(MDSCluster): self.set_max_mds(new_max_mds) return self.wait_for_daemons() + def run_scrub(self, cmd, rank=0): + return self.rank_tell(["scrub"] + cmd, rank) + def get_scrub_status(self, rank=0): - return self.rank_tell(["scrub", "status"], rank) + return self.run_scrub(["status"], rank) def wait_until_scrub_complete(self, result=None, tag=None, rank=0, sleep=30, timeout=300): # time out after "timeout" seconds and assume as done diff --git a/qa/tasks/cephfs/test_damage.py b/qa/tasks/cephfs/test_damage.py index 43ffbfe882d8..c6067dc32062 100644 --- a/qa/tasks/cephfs/test_damage.py +++ b/qa/tasks/cephfs/test_damage.py @@ -451,7 +451,7 @@ class TestDamage(CephFSTestCase): self.mount_a.umount_wait() # Now repair the stats - scrub_json = self.fs.rank_tell(["scrub", "start", "/subdir", "repair"]) + scrub_json = self.fs.run_scrub(["start", "/subdir", "repair"]) log.info(json.dumps(scrub_json, indent=2)) self.assertNotEqual(scrub_json, None) diff --git a/qa/tasks/cephfs/test_data_scan.py b/qa/tasks/cephfs/test_data_scan.py index e3616b6c5fd5..0e1c0bd72739 100644 --- a/qa/tasks/cephfs/test_data_scan.py +++ b/qa/tasks/cephfs/test_data_scan.py @@ -509,7 +509,7 @@ class TestDataScan(CephFSTestCase): # run scrub to update and make sure rstat.rbytes info in subdir inode and dirfrag # are matched - out_json = self.fs.rank_tell(["scrub", "start", "/subdir", "repair", "recursive"]) + out_json = self.fs.run_scrub(["start", "/subdir", "repair", "recursive"]) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) diff --git a/qa/tasks/cephfs/test_forward_scrub.py b/qa/tasks/cephfs/test_forward_scrub.py index 97d68df3aa30..59ca24845095 100644 --- a/qa/tasks/cephfs/test_forward_scrub.py +++ b/qa/tasks/cephfs/test_forward_scrub.py @@ -234,7 +234,7 @@ class TestForwardScrub(CephFSTestCase): self.mount_a.umount_wait() with self.assert_cluster_log("inode table repaired", invert_match=True): - out_json = self.fs.rank_tell(["scrub", "start", "/", "repair", "recursive"]) + out_json = self.fs.run_scrub(["start", "/", "repair", "recursive"]) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) @@ -259,7 +259,7 @@ class TestForwardScrub(CephFSTestCase): self.fs.wait_for_daemons() with self.assert_cluster_log("inode table repaired"): - out_json = self.fs.rank_tell(["scrub", "start", "/", "repair", "recursive"]) + out_json = self.fs.run_scrub(["start", "/", "repair", "recursive"]) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) @@ -292,7 +292,7 @@ class TestForwardScrub(CephFSTestCase): "oh i'm sorry did i overwrite your xattr?") with self.assert_cluster_log("bad backtrace on inode"): - out_json = self.fs.rank_tell(["scrub", "start", "/", "repair", "recursive"]) + out_json = self.fs.run_scrub(["start", "/", "repair", "recursive"]) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) diff --git a/qa/tasks/cephfs/test_multimds_misc.py b/qa/tasks/cephfs/test_multimds_misc.py index 2d7d01ed6280..921a1ffa256f 100644 --- a/qa/tasks/cephfs/test_multimds_misc.py +++ b/qa/tasks/cephfs/test_multimds_misc.py @@ -86,7 +86,7 @@ class TestScrub2(CephFSTestCase): file_obj_name = "{0:x}.00000000".format(ino) self.fs.rados(["rmxattr", file_obj_name, "parent"]) - out_json = self.fs.rank_tell(["scrub", "start", "/d1/d2/d3", "recursive", "force"], 0) + out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) @@ -129,10 +129,10 @@ class TestScrub2(CephFSTestCase): file_obj_name = "{0:x}.00000000".format(ino) self.fs.rados(["rmxattr", file_obj_name, "parent"]) - out_json = self.fs.rank_tell(["scrub", "start", "/d1/d2/d3", "recursive", "force"], 0) + out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0) self.assertNotEqual(out_json, None) - res = self.fs.rank_tell(["scrub", "abort"]) + res = self.fs.run_scrub(["abort"]) self.assertEqual(res['return_code'], 0) # Abort and verify in both mdss. We also check the status in rank 0 mds because @@ -154,10 +154,10 @@ class TestScrub2(CephFSTestCase): file_obj_name = "{0:x}.00000000".format(ino) self.fs.rados(["rmxattr", file_obj_name, "parent"]) - out_json = self.fs.rank_tell(["scrub", "start", "/d1/d2/d3", "recursive", "force"], 0) + out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0) self.assertNotEqual(out_json, None) - res = self.fs.rank_tell(["scrub", "pause"]) + res = self.fs.run_scrub(["pause"]) self.assertEqual(res['return_code'], 0) self.wait_until_true(lambda: "PAUSED" in self.fs.get_scrub_status(1)['status'] @@ -168,7 +168,7 @@ class TestScrub2(CephFSTestCase): self.assertTrue(checked) # resume and verify - res = self.fs.rank_tell(["scrub", "resume"]) + res = self.fs.run_scrub(["resume"]) self.assertEqual(res['return_code'], 0) self.wait_until_true(lambda: not("PAUSED" in self.fs.get_scrub_status(1)['status']) @@ -187,10 +187,10 @@ class TestScrub2(CephFSTestCase): file_obj_name = "{0:x}.00000000".format(ino) self.fs.rados(["rmxattr", file_obj_name, "parent"]) - out_json = self.fs.rank_tell(["scrub", "start", "/d1/d2/d3", "recursive", "force"], 0) + out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0) self.assertNotEqual(out_json, None) - res = self.fs.rank_tell(["scrub", "pause"]) + res = self.fs.run_scrub(["pause"]) self.assertEqual(res['return_code'], 0) self.wait_until_true(lambda: "PAUSED" in self.fs.get_scrub_status(1)['status'] @@ -200,7 +200,7 @@ class TestScrub2(CephFSTestCase): checked = self._check_task_status("paused") self.assertTrue(checked) - res = self.fs.rank_tell(["scrub", "abort"]) + res = self.fs.run_scrub(["abort"]) self.assertEqual(res['return_code'], 0) self.wait_until_true(lambda: "PAUSED" in self.fs.get_scrub_status(1)['status'] @@ -215,7 +215,7 @@ class TestScrub2(CephFSTestCase): self.assertTrue(checked) # resume and verify - res = self.fs.rank_tell(["scrub", "resume"]) + res = self.fs.run_scrub(["resume"]) self.assertEqual(res['return_code'], 0) self.wait_until_true(lambda: not("PAUSED" in self.fs.get_scrub_status(1)['status']) diff --git a/qa/tasks/cephfs/test_scrub.py b/qa/tasks/cephfs/test_scrub.py index 896ea0480df8..01e9dc8f6f3f 100644 --- a/qa/tasks/cephfs/test_scrub.py +++ b/qa/tasks/cephfs/test_scrub.py @@ -104,7 +104,7 @@ class DupInodeWorkload(Workload): self._filesystem.wait_for_daemons() def validate(self): - out_json = self._filesystem.rank_tell(["scrub", "start", "/", "recursive", "repair"]) + out_json = self._filesystem.run_scrub(["start", "/", "recursive", "repair"]) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self._filesystem.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) @@ -135,7 +135,7 @@ class TestScrub(CephFSTestCase): # Apply any data damage the workload wants workload.damage() - out_json = self.fs.rank_tell(["scrub", "start", "/", "recursive", "repair"]) + out_json = self.fs.run_scrub(["start", "/", "recursive", "repair"]) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) diff --git a/qa/tasks/cephfs/test_scrub_checks.py b/qa/tasks/cephfs/test_scrub_checks.py index f6698ede2ae5..0d534514f351 100644 --- a/qa/tasks/cephfs/test_scrub_checks.py +++ b/qa/tasks/cephfs/test_scrub_checks.py @@ -21,13 +21,13 @@ class TestScrubControls(CephFSTestCase): CLIENTS_REQUIRED = 1 def _abort_scrub(self, expected): - res = self.fs.rank_tell(["scrub", "abort"]) + res = self.fs.run_scrub(["abort"]) self.assertEqual(res['return_code'], expected) def _pause_scrub(self, expected): - res = self.fs.rank_tell(["scrub", "pause"]) + res = self.fs.run_scrub(["pause"]) self.assertEqual(res['return_code'], expected) def _resume_scrub(self, expected): - res = self.fs.rank_tell(["scrub", "resume"]) + res = self.fs.run_scrub(["resume"]) self.assertEqual(res['return_code'], expected) def _check_task_status(self, expected_status, timo=120): """ check scrub status for current active mds in ceph status """ @@ -70,7 +70,7 @@ done self.create_scrub_data(test_dir) - out_json = self.fs.rank_tell(["scrub", "start", abs_test_path, "recursive"]) + out_json = self.fs.run_scrub(["start", abs_test_path, "recursive"]) self.assertNotEqual(out_json, None) # abort and verify @@ -91,7 +91,7 @@ done self.create_scrub_data(test_dir) - out_json = self.fs.rank_tell(["scrub", "start", abs_test_path, "recursive"]) + out_json = self.fs.run_scrub(["start", abs_test_path, "recursive"]) self.assertNotEqual(out_json, None) # pause and verify @@ -116,7 +116,7 @@ done self.create_scrub_data(test_dir) - out_json = self.fs.rank_tell(["scrub", "start", abs_test_path, "recursive"]) + out_json = self.fs.run_scrub(["start", abs_test_path, "recursive"]) self.assertNotEqual(out_json, None) # pause and verify @@ -154,7 +154,7 @@ done self.create_scrub_data(test_dir) - out_json = self.fs.rank_tell(["scrub", "start", abs_test_path, "recursive"]) + out_json = self.fs.run_scrub(["start", abs_test_path, "recursive"]) self.assertNotEqual(out_json, None) # pause and verify diff --git a/qa/tasks/cephfs_upgrade_snap.py b/qa/tasks/cephfs_upgrade_snap.py index 5cc388a05dc9..1b0a737a7b9c 100644 --- a/qa/tasks/cephfs_upgrade_snap.py +++ b/qa/tasks/cephfs_upgrade_snap.py @@ -24,14 +24,14 @@ def task(ctx, config): mds_map = fs.get_mds_map() assert(mds_map['max_mds'] == 1) - json = fs.rank_tell(["scrub", "start", "/", "force", "recursive", "repair"]) + json = fs.run_scrub(["start", "/", "force", "recursive", "repair"]) if not json or json['return_code'] == 0: assert(fs.wait_until_scrub_complete(tag=json["scrub_tag"]) == True) log.info("scrub / completed") else: log.info("scrub / failed: {}".format(json)) - json = fs.rank_tell(["scrub", "start", "~mdsdir", "force", "recursive", "repair"]) + json = fs.run_scrub(["start", "~mdsdir", "force", "recursive", "repair"]) if not json or json['return_code'] == 0: assert(fs.wait_until_scrub_complete(tag=json["scrub_tag"]) == True) log.info("scrub ~mdsdir completed") diff --git a/qa/tasks/fwd_scrub.py b/qa/tasks/fwd_scrub.py index c8866c1901a4..a7ae1c4ed74a 100644 --- a/qa/tasks/fwd_scrub.py +++ b/qa/tasks/fwd_scrub.py @@ -62,7 +62,7 @@ class ForwardScrubber(Thrasher, Greenlet): def _scrub(self, path="/", recursive=True): self.logger.info(f"scrubbing fs: {self.fs.name}") recopt = ["recursive", "force"] if recursive else ["force"] - out_json = self.fs.rank_tell(["scrub", "start", path] + recopt) + out_json = self.fs.run_scrub(["start", path] + recopt) assert out_json is not None tag = out_json['scrub_tag']