raise RuntimeError("rank {0} failed to reach desired subtree state".format(rank)) from e
def _wait_until_scrub_complete(self, path="/", recursive=True, timeout=100):
- out_json = self.fs.rank_tell(["scrub", "start", path] + ["recursive"] if recursive else [])
+ out_json = self.fs.run_scrub(["start", path] + ["recursive"] if recursive else [])
if not self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"],
sleep=10, timeout=timeout):
log.info("timed out waiting for scrub to complete")
self.set_max_mds(new_max_mds)
return self.wait_for_daemons()
+ def run_scrub(self, cmd, rank=0):
+ return self.rank_tell(["scrub"] + cmd, rank)
+
def get_scrub_status(self, rank=0):
- return self.rank_tell(["scrub", "status"], rank)
+ return self.run_scrub(["status"], rank)
def wait_until_scrub_complete(self, result=None, tag=None, rank=0, sleep=30, timeout=300):
# time out after "timeout" seconds and assume as done
self.mount_a.umount_wait()
# Now repair the stats
- scrub_json = self.fs.rank_tell(["scrub", "start", "/subdir", "repair"])
+ scrub_json = self.fs.run_scrub(["start", "/subdir", "repair"])
log.info(json.dumps(scrub_json, indent=2))
self.assertNotEqual(scrub_json, None)
# run scrub to update and make sure rstat.rbytes info in subdir inode and dirfrag
# are matched
- out_json = self.fs.rank_tell(["scrub", "start", "/subdir", "repair", "recursive"])
+ out_json = self.fs.run_scrub(["start", "/subdir", "repair", "recursive"])
self.assertNotEqual(out_json, None)
self.assertEqual(out_json["return_code"], 0)
self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
self.mount_a.umount_wait()
with self.assert_cluster_log("inode table repaired", invert_match=True):
- out_json = self.fs.rank_tell(["scrub", "start", "/", "repair", "recursive"])
+ out_json = self.fs.run_scrub(["start", "/", "repair", "recursive"])
self.assertNotEqual(out_json, None)
self.assertEqual(out_json["return_code"], 0)
self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
self.fs.wait_for_daemons()
with self.assert_cluster_log("inode table repaired"):
- out_json = self.fs.rank_tell(["scrub", "start", "/", "repair", "recursive"])
+ out_json = self.fs.run_scrub(["start", "/", "repair", "recursive"])
self.assertNotEqual(out_json, None)
self.assertEqual(out_json["return_code"], 0)
self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
"oh i'm sorry did i overwrite your xattr?")
with self.assert_cluster_log("bad backtrace on inode"):
- out_json = self.fs.rank_tell(["scrub", "start", "/", "repair", "recursive"])
+ out_json = self.fs.run_scrub(["start", "/", "repair", "recursive"])
self.assertNotEqual(out_json, None)
self.assertEqual(out_json["return_code"], 0)
self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
file_obj_name = "{0:x}.00000000".format(ino)
self.fs.rados(["rmxattr", file_obj_name, "parent"])
- out_json = self.fs.rank_tell(["scrub", "start", "/d1/d2/d3", "recursive", "force"], 0)
+ out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0)
self.assertNotEqual(out_json, None)
self.assertEqual(out_json["return_code"], 0)
self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
file_obj_name = "{0:x}.00000000".format(ino)
self.fs.rados(["rmxattr", file_obj_name, "parent"])
- out_json = self.fs.rank_tell(["scrub", "start", "/d1/d2/d3", "recursive", "force"], 0)
+ out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0)
self.assertNotEqual(out_json, None)
- res = self.fs.rank_tell(["scrub", "abort"])
+ res = self.fs.run_scrub(["abort"])
self.assertEqual(res['return_code'], 0)
# Abort and verify in both mdss. We also check the status in rank 0 mds because
file_obj_name = "{0:x}.00000000".format(ino)
self.fs.rados(["rmxattr", file_obj_name, "parent"])
- out_json = self.fs.rank_tell(["scrub", "start", "/d1/d2/d3", "recursive", "force"], 0)
+ out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0)
self.assertNotEqual(out_json, None)
- res = self.fs.rank_tell(["scrub", "pause"])
+ res = self.fs.run_scrub(["pause"])
self.assertEqual(res['return_code'], 0)
self.wait_until_true(lambda: "PAUSED" in self.fs.get_scrub_status(1)['status']
self.assertTrue(checked)
# resume and verify
- res = self.fs.rank_tell(["scrub", "resume"])
+ res = self.fs.run_scrub(["resume"])
self.assertEqual(res['return_code'], 0)
self.wait_until_true(lambda: not("PAUSED" in self.fs.get_scrub_status(1)['status'])
file_obj_name = "{0:x}.00000000".format(ino)
self.fs.rados(["rmxattr", file_obj_name, "parent"])
- out_json = self.fs.rank_tell(["scrub", "start", "/d1/d2/d3", "recursive", "force"], 0)
+ out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0)
self.assertNotEqual(out_json, None)
- res = self.fs.rank_tell(["scrub", "pause"])
+ res = self.fs.run_scrub(["pause"])
self.assertEqual(res['return_code'], 0)
self.wait_until_true(lambda: "PAUSED" in self.fs.get_scrub_status(1)['status']
checked = self._check_task_status("paused")
self.assertTrue(checked)
- res = self.fs.rank_tell(["scrub", "abort"])
+ res = self.fs.run_scrub(["abort"])
self.assertEqual(res['return_code'], 0)
self.wait_until_true(lambda: "PAUSED" in self.fs.get_scrub_status(1)['status']
self.assertTrue(checked)
# resume and verify
- res = self.fs.rank_tell(["scrub", "resume"])
+ res = self.fs.run_scrub(["resume"])
self.assertEqual(res['return_code'], 0)
self.wait_until_true(lambda: not("PAUSED" in self.fs.get_scrub_status(1)['status'])
self._filesystem.wait_for_daemons()
def validate(self):
- out_json = self._filesystem.rank_tell(["scrub", "start", "/", "recursive", "repair"])
+ out_json = self._filesystem.run_scrub(["start", "/", "recursive", "repair"])
self.assertNotEqual(out_json, None)
self.assertEqual(out_json["return_code"], 0)
self.assertEqual(self._filesystem.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
# Apply any data damage the workload wants
workload.damage()
- out_json = self.fs.rank_tell(["scrub", "start", "/", "recursive", "repair"])
+ out_json = self.fs.run_scrub(["start", "/", "recursive", "repair"])
self.assertNotEqual(out_json, None)
self.assertEqual(out_json["return_code"], 0)
self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
CLIENTS_REQUIRED = 1
def _abort_scrub(self, expected):
- res = self.fs.rank_tell(["scrub", "abort"])
+ res = self.fs.run_scrub(["abort"])
self.assertEqual(res['return_code'], expected)
def _pause_scrub(self, expected):
- res = self.fs.rank_tell(["scrub", "pause"])
+ res = self.fs.run_scrub(["pause"])
self.assertEqual(res['return_code'], expected)
def _resume_scrub(self, expected):
- res = self.fs.rank_tell(["scrub", "resume"])
+ res = self.fs.run_scrub(["resume"])
self.assertEqual(res['return_code'], expected)
def _check_task_status(self, expected_status, timo=120):
""" check scrub status for current active mds in ceph status """
self.create_scrub_data(test_dir)
- out_json = self.fs.rank_tell(["scrub", "start", abs_test_path, "recursive"])
+ out_json = self.fs.run_scrub(["start", abs_test_path, "recursive"])
self.assertNotEqual(out_json, None)
# abort and verify
self.create_scrub_data(test_dir)
- out_json = self.fs.rank_tell(["scrub", "start", abs_test_path, "recursive"])
+ out_json = self.fs.run_scrub(["start", abs_test_path, "recursive"])
self.assertNotEqual(out_json, None)
# pause and verify
self.create_scrub_data(test_dir)
- out_json = self.fs.rank_tell(["scrub", "start", abs_test_path, "recursive"])
+ out_json = self.fs.run_scrub(["start", abs_test_path, "recursive"])
self.assertNotEqual(out_json, None)
# pause and verify
self.create_scrub_data(test_dir)
- out_json = self.fs.rank_tell(["scrub", "start", abs_test_path, "recursive"])
+ out_json = self.fs.run_scrub(["start", abs_test_path, "recursive"])
self.assertNotEqual(out_json, None)
# pause and verify
mds_map = fs.get_mds_map()
assert(mds_map['max_mds'] == 1)
- json = fs.rank_tell(["scrub", "start", "/", "force", "recursive", "repair"])
+ json = fs.run_scrub(["start", "/", "force", "recursive", "repair"])
if not json or json['return_code'] == 0:
assert(fs.wait_until_scrub_complete(tag=json["scrub_tag"]) == True)
log.info("scrub / completed")
else:
log.info("scrub / failed: {}".format(json))
- json = fs.rank_tell(["scrub", "start", "~mdsdir", "force", "recursive", "repair"])
+ json = fs.run_scrub(["start", "~mdsdir", "force", "recursive", "repair"])
if not json or json['return_code'] == 0:
assert(fs.wait_until_scrub_complete(tag=json["scrub_tag"]) == True)
log.info("scrub ~mdsdir completed")
def _scrub(self, path="/", recursive=True):
self.logger.info(f"scrubbing fs: {self.fs.name}")
recopt = ["recursive", "force"] if recursive else ["force"]
- out_json = self.fs.rank_tell(["scrub", "start", path] + recopt)
+ out_json = self.fs.run_scrub(["start", path] + recopt)
assert out_json is not None
tag = out_json['scrub_tag']