qa: update scrub start code to use comma sep scrubopts

The documentation specifies this in [1] and yet we were using (I
believe) an older syntax:

    ceph tell mds.foo:0 scrub start / recursive force

instead of

    ceph tell mds.foo:0 scrub start / recursive,force

Oddly the former works at least as recently as in [2]:

    2021-06-03T07:11:42.071 DEBUG:teuthology.orchestra.run.smithi025:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub start / recursive force
    ...
    2021-06-03T07:11:42.268 INFO:teuthology.orchestra.run.smithi025.stdout:{
    2021-06-03T07:11:42.268 INFO:teuthology.orchestra.run.smithi025.stdout:    "return_code": 0,
    2021-06-03T07:11:42.268 INFO:teuthology.orchestra.run.smithi025.stdout:    "scrub_tag": "cf7a74b2-3eb2-4657-9274-ea504b1ebf8f",
    2021-06-03T07:11:42.269 INFO:teuthology.orchestra.run.smithi025.stdout:    "mode": "asynchronous"
    2021-06-03T07:11:42.269 INFO:teuthology.orchestra.run.smithi025.stdout:}

[1] https://docs.ceph.com/en/latest/cephfs/scrub/
[2] /ceph/teuthology-archive/pdonnell-2021-06-03_03:40:33-fs-wip-pdonnell-testing-20210603.020013-distro-basic-smithi/6148097/teuthology.log

Fixes: https://tracker.ceph.com/issues/51146
See-also: https://tracker.ceph.com/issues/51145
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 0f505dc299fd10f27bab74b02356c1164814d0f5)

diff --git a/qa/tasks/cephfs/test_data_scan.py b/qa/tasks/cephfs/test_data_scan.py
index 7d50f1003ce6029d6e3855986e8d429df13ba751..5552b69c3e7ebaf726debc7ed68a9d0cbc67ae85 100644 (file)
--- a/qa/tasks/cephfs/test_data_scan.py
+++ b/qa/tasks/cephfs/test_data_scan.py
@@ -504,7 +504,7 @@ class TestDataScan(CephFSTestCase):
 
         # run scrub to update and make sure rstat.rbytes info in subdir inode and dirfrag
         # are matched
-        out_json = self.fs.run_scrub(["start", "/subdir", "repair", "recursive"])
+        out_json = self.fs.run_scrub(["start", "/subdir", "repair,recursive"])
         self.assertNotEqual(out_json, None)
         self.assertEqual(out_json["return_code"], 0)
         self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)

diff --git a/qa/tasks/cephfs/test_forward_scrub.py b/qa/tasks/cephfs/test_forward_scrub.py
index 19eb4afaf2a53f6015e36041a8fdccc8329c43a3..82630e069587b7105c524f93add50e8237eac45b 100644 (file)
--- a/qa/tasks/cephfs/test_forward_scrub.py
+++ b/qa/tasks/cephfs/test_forward_scrub.py
@@ -232,7 +232,7 @@ class TestForwardScrub(CephFSTestCase):
         self.mount_a.umount_wait()
 
         with self.assert_cluster_log("inode table repaired", invert_match=True):
-            out_json = self.fs.run_scrub(["start", "/", "repair", "recursive"])
+            out_json = self.fs.run_scrub(["start", "/", "repair,recursive"])
             self.assertNotEqual(out_json, None)
             self.assertEqual(out_json["return_code"], 0)
             self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
@@ -256,7 +256,7 @@ class TestForwardScrub(CephFSTestCase):
         self.fs.wait_for_daemons()
 
         with self.assert_cluster_log("inode table repaired"):
-            out_json = self.fs.run_scrub(["start", "/", "repair", "recursive"])
+            out_json = self.fs.run_scrub(["start", "/", "repair,recursive"])
             self.assertNotEqual(out_json, None)
             self.assertEqual(out_json["return_code"], 0)
             self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
@@ -289,7 +289,7 @@ class TestForwardScrub(CephFSTestCase):
                                   "oh i'm sorry did i overwrite your xattr?")
 
         with self.assert_cluster_log("bad backtrace on inode"):
-            out_json = self.fs.run_scrub(["start", "/", "repair", "recursive"])
+            out_json = self.fs.run_scrub(["start", "/", "repair,recursive"])
             self.assertNotEqual(out_json, None)
             self.assertEqual(out_json["return_code"], 0)
             self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)

diff --git a/qa/tasks/cephfs/test_multimds_misc.py b/qa/tasks/cephfs/test_multimds_misc.py
index 77f15ec2dd5c541f2b75345cefc9aa21885c402d..3c464e91d0a28a69210f205ab98194b7c198f49e 100644 (file)
--- a/qa/tasks/cephfs/test_multimds_misc.py
+++ b/qa/tasks/cephfs/test_multimds_misc.py
@@ -97,7 +97,7 @@ class TestScrub2(CephFSTestCase):
             file_obj_name = "{0:x}.00000000".format(ino)
             self.fs.radosm(["rmxattr", file_obj_name, "parent"])
 
-        out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0)
+        out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive,force"], 0)
         self.assertNotEqual(out_json, None)
         self.assertEqual(out_json["return_code"], 0)
         self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
@@ -140,7 +140,7 @@ class TestScrub2(CephFSTestCase):
             file_obj_name = "{0:x}.00000000".format(ino)
             self.fs.radosm(["rmxattr", file_obj_name, "parent"])
 
-        out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0)
+        out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive,force"], 0)
         self.assertNotEqual(out_json, None)
         
         res = self.fs.run_scrub(["abort"])
@@ -163,7 +163,7 @@ class TestScrub2(CephFSTestCase):
             file_obj_name = "{0:x}.00000000".format(ino)
             self.fs.radosm(["rmxattr", file_obj_name, "parent"])
 
-        out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0)
+        out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive,force"], 0)
         self.assertNotEqual(out_json, None)
 
         res = self.fs.run_scrub(["pause"])
@@ -192,7 +192,7 @@ class TestScrub2(CephFSTestCase):
             file_obj_name = "{0:x}.00000000".format(ino)
             self.fs.radosm(["rmxattr", file_obj_name, "parent"])
 
-        out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0)
+        out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive,force"], 0)
         self.assertNotEqual(out_json, None)
 
         res = self.fs.run_scrub(["pause"])

diff --git a/qa/tasks/cephfs/test_recovery_pool.py b/qa/tasks/cephfs/test_recovery_pool.py
index 9742bfa4768a1e728f64a95a035bde0b12c714b1..9926b3670cac3b4bd30b2fd6b28d99c4384587c3 100644 (file)
--- a/qa/tasks/cephfs/test_recovery_pool.py
+++ b/qa/tasks/cephfs/test_recovery_pool.py
@@ -180,7 +180,7 @@ class TestRecoveryPool(CephFSTestCase):
         for rank in self.recovery_fs.get_ranks(status=status):
             self.fs.mon_manager.raw_cluster_cmd('tell', "mds." + rank['name'],
                                                 'injectargs', '--debug-mds=20')
-            self.fs.rank_tell(['scrub', 'start', '/', 'recursive', 'repair'], rank=rank['rank'], status=status)
+            self.fs.rank_tell(['scrub', 'start', '/', 'recursive,repair'], rank=rank['rank'], status=status)
         log.info(str(self.mds_cluster.status()))
 
         # Mount a client

diff --git a/qa/tasks/cephfs/test_scrub.py b/qa/tasks/cephfs/test_scrub.py
index 5d571460a1e57d382b6c5fa9b16316aa6cf8d3a0..dd7c11af50b39a6517b2267d1f5e88c4f99a8e6a 100644 (file)
--- a/qa/tasks/cephfs/test_scrub.py
+++ b/qa/tasks/cephfs/test_scrub.py
@@ -103,7 +103,7 @@ class DupInodeWorkload(Workload):
         self._filesystem.wait_for_daemons()
 
     def validate(self):
-        out_json = self._filesystem.run_scrub(["start", "/", "recursive", "repair"])
+        out_json = self._filesystem.run_scrub(["start", "/", "recursive,repair"])
         self.assertNotEqual(out_json, None)
         self.assertEqual(out_json["return_code"], 0)
         self.assertEqual(self._filesystem.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
@@ -134,7 +134,7 @@ class TestScrub(CephFSTestCase):
         # Apply any data damage the workload wants
         workload.damage()
 
-        out_json = self.fs.run_scrub(["start", "/", "recursive", "repair"])
+        out_json = self.fs.run_scrub(["start", "/", "recursive,repair"])
         self.assertNotEqual(out_json, None)
         self.assertEqual(out_json["return_code"], 0)
         self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)

diff --git a/qa/tasks/fwd_scrub.py b/qa/tasks/fwd_scrub.py
index a7ae1c4ed74a485e18ac03df292f882979dd7808..44fd97baa0a6377fd0b7818af3cb3c602bbba7ed 100644 (file)
--- a/qa/tasks/fwd_scrub.py
+++ b/qa/tasks/fwd_scrub.py
@@ -61,8 +61,10 @@ class ForwardScrubber(Thrasher, Greenlet):
 
     def _scrub(self, path="/", recursive=True):
         self.logger.info(f"scrubbing fs: {self.fs.name}")
-        recopt = ["recursive", "force"] if recursive else ["force"]
-        out_json = self.fs.run_scrub(["start", path] + recopt)
+        scrubopts = ["force"]
+        if recursive:
+            scrubopts.append("recursive")
+        out_json = self.fs.run_scrub(["start", path, ",".join(scrubopts)])
         assert out_json is not None
 
         tag = out_json['scrub_tag']