From: Xiubo Li Date: Wed, 23 Dec 2020 01:54:52 +0000 (+0800) Subject: qa: wait the scrub task to complete X-Git-Tag: v16.2.0~20^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=fbbf01b1ba92ef4c01cd9a0e5235168e3bff7b70;p=ceph.git qa: wait the scrub task to complete Signed-off-by: Xiubo Li (cherry picked from commit 7b857c63db8be8ecfdd96bf101cdae09226cba03) --- diff --git a/qa/tasks/cephfs/test_data_scan.py b/qa/tasks/cephfs/test_data_scan.py index 7b6459de923..e3616b6c5fd 100644 --- a/qa/tasks/cephfs/test_data_scan.py +++ b/qa/tasks/cephfs/test_data_scan.py @@ -511,6 +511,8 @@ class TestDataScan(CephFSTestCase): # are matched out_json = self.fs.rank_tell(["scrub", "start", "/subdir", "repair", "recursive"]) self.assertNotEqual(out_json, None) + self.assertEqual(out_json["return_code"], 0) + self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) # Remove the whole 'sudbdir' directory self.mount_a.run_shell(["rm", "-rf", "subdir/"]) diff --git a/qa/tasks/cephfs/test_forward_scrub.py b/qa/tasks/cephfs/test_forward_scrub.py index 9ca3c4de790..97d68df3aa3 100644 --- a/qa/tasks/cephfs/test_forward_scrub.py +++ b/qa/tasks/cephfs/test_forward_scrub.py @@ -236,6 +236,8 @@ class TestForwardScrub(CephFSTestCase): with self.assert_cluster_log("inode table repaired", invert_match=True): out_json = self.fs.rank_tell(["scrub", "start", "/", "repair", "recursive"]) self.assertNotEqual(out_json, None) + self.assertEqual(out_json["return_code"], 0) + self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) self.mds_cluster.mds_stop() self.mds_cluster.mds_fail() @@ -259,6 +261,8 @@ class TestForwardScrub(CephFSTestCase): with self.assert_cluster_log("inode table repaired"): out_json = self.fs.rank_tell(["scrub", "start", "/", "repair", "recursive"]) self.assertNotEqual(out_json, None) + self.assertEqual(out_json["return_code"], 0) + self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) self.mds_cluster.mds_stop() table_text = self.fs.table_tool(["0", "show", "inode"]) @@ -290,6 +294,9 @@ class TestForwardScrub(CephFSTestCase): with self.assert_cluster_log("bad backtrace on inode"): out_json = self.fs.rank_tell(["scrub", "start", "/", "repair", "recursive"]) self.assertNotEqual(out_json, None) + self.assertEqual(out_json["return_code"], 0) + self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) + self.fs.mds_asok(["flush", "journal"]) backtrace = self.fs.read_backtrace(file_ino) self.assertEqual(['alpha', 'parent_a'], diff --git a/qa/tasks/cephfs/test_multimds_misc.py b/qa/tasks/cephfs/test_multimds_misc.py index eecd2e48840..ee93515d48d 100644 --- a/qa/tasks/cephfs/test_multimds_misc.py +++ b/qa/tasks/cephfs/test_multimds_misc.py @@ -13,9 +13,6 @@ class TestScrub2(CephFSTestCase): def _get_scrub_status(self, rank=0): return self.fs.rank_tell(["scrub", "status"], rank) - def _wait_until_scrubbed(self, timeout): - self.wait_until_true(lambda: "no active" in self._get_scrub_status()['status'], timeout) - def _check_task_status_na(self, timo=120): """ check absence of scrub status in ceph status """ with safe_while(sleep=1, tries=120, action='wait for task status') as proceed: @@ -72,8 +69,10 @@ class TestScrub2(CephFSTestCase): inos = self._find_path_inos('d1/d2/d3/') tag = "tag123" - self.fs.rank_tell(["tag", "path", "/d1/d2/d3", tag], 0) - self._wait_until_scrubbed(30); + out_json = self.fs.rank_tell(["tag", "path", "/d1/d2/d3", tag], 0) + self.assertNotEqual(out_json, None) + self.assertEqual(out_json["return_code"], 0) + self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) def assertTagged(ino): file_obj_name = "{0:x}.00000000".format(ino) @@ -90,8 +89,10 @@ class TestScrub2(CephFSTestCase): file_obj_name = "{0:x}.00000000".format(ino) self.fs.rados(["rmxattr", file_obj_name, "parent"]) - self.fs.rank_tell(["scrub", "start", "/d1/d2/d3", "recursive", "force"], 0) - self._wait_until_scrubbed(30); + out_json = self.fs.rank_tell(["scrub", "start", "/d1/d2/d3", "recursive", "force"], 0) + self.assertNotEqual(out_json, None) + self.assertEqual(out_json["return_code"], 0) + self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) def _check_damage(mds_rank, inos): all_damage = self.fs.rank_tell(["damage", "ls"], mds_rank) diff --git a/qa/tasks/cephfs/test_scrub.py b/qa/tasks/cephfs/test_scrub.py index 1875b5f34fd..896ea0480df 100644 --- a/qa/tasks/cephfs/test_scrub.py +++ b/qa/tasks/cephfs/test_scrub.py @@ -106,6 +106,8 @@ class DupInodeWorkload(Workload): def validate(self): out_json = self._filesystem.rank_tell(["scrub", "start", "/", "recursive", "repair"]) self.assertNotEqual(out_json, None) + self.assertEqual(out_json["return_code"], 0) + self.assertEqual(self._filesystem.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) self.assertTrue(self._filesystem.are_daemons_healthy()) return self._errors @@ -135,6 +137,8 @@ class TestScrub(CephFSTestCase): out_json = self.fs.rank_tell(["scrub", "start", "/", "recursive", "repair"]) self.assertNotEqual(out_json, None) + self.assertEqual(out_json["return_code"], 0) + self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) # See that the files are present and correct errors = workload.validate() @@ -162,7 +166,7 @@ class TestScrub(CephFSTestCase): That scrubbing new files does not lead to errors """ workload.create_files(1000) - self._wait_until_scrub_complete() + self.fs.wait_until_scrub_complete() self.assertEqual(self._get_damage_count(), 0) def test_scrub_backtrace_for_new_files(self): diff --git a/qa/tasks/cephfs_upgrade_snap.py b/qa/tasks/cephfs_upgrade_snap.py index 1708d43cfe7..5cc388a05dc 100644 --- a/qa/tasks/cephfs_upgrade_snap.py +++ b/qa/tasks/cephfs_upgrade_snap.py @@ -26,12 +26,14 @@ def task(ctx, config): json = fs.rank_tell(["scrub", "start", "/", "force", "recursive", "repair"]) if not json or json['return_code'] == 0: + assert(fs.wait_until_scrub_complete(tag=json["scrub_tag"]) == True) log.info("scrub / completed") else: log.info("scrub / failed: {}".format(json)) json = fs.rank_tell(["scrub", "start", "~mdsdir", "force", "recursive", "repair"]) if not json or json['return_code'] == 0: + assert(fs.wait_until_scrub_complete(tag=json["scrub_tag"]) == True) log.info("scrub ~mdsdir completed") else: log.info("scrub / failed: {}".format(json))