From db9825d3142d2f886ef4719adfbac51209136300 Mon Sep 17 00:00:00 2001 From: Jos Collin Date: Mon, 11 Mar 2024 16:11:54 +0530 Subject: [PATCH] qa: make all replayer threads busy and query 'syncing' status * Make all replayer threads busy and then query for 'syncing' state instead of just fetching the current status. * Dropped 'current_syncing_snap' check, as it's not compulsory for this test. The actual intension is to make threads in 'syncing' status and 'current_syncing_snap' check is not necessary for that. * Drop 'snaps_deleted' metrics check in test_cephfs_mirror_cancel_mirroring_and_readd. test_cephfs_mirror_cancel_mirroring_and_readd primarily focusses on the synchronization of the newly added directory paths post removal of the previously added/syncing directory paths. So checking of 'snaps_deleted' metrics is unnecessary here. * Wait for more time to finish the new snapshot creations and the sync backoff. We need to wait for more time in test_cephfs_mirror_cancel_mirroring_and_readd, as the test makes all replayer threads busy. Fixes: https://tracker.ceph.com/issues/64711 Signed-off-by: Jos Collin --- qa/tasks/cephfs/test_mirroring.py | 65 ++++++++++++++++++------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/qa/tasks/cephfs/test_mirroring.py b/qa/tasks/cephfs/test_mirroring.py index 2f9ebe6b1d592..336fc071a4c60 100644 --- a/qa/tasks/cephfs/test_mirroring.py +++ b/qa/tasks/cephfs/test_mirroring.py @@ -1361,7 +1361,7 @@ class TestMirroring(CephFSTestCase): self.mount_b.umount_wait() self.mount_b.mount_wait(cephfs_name=self.secondary_fs_name) - # create a bunch of files in a directory to snap + # create some large files in 3 directories to snap self.mount_a.run_shell(["mkdir", "d0"]) self.mount_a.run_shell(["mkdir", "d1"]) self.mount_a.run_shell(["mkdir", "d2"]) @@ -1384,30 +1384,38 @@ class TestMirroring(CephFSTestCase): vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0] # take snapshots log.debug('taking snapshots') - self.mount_a.run_shell(["mkdir", "d0/.snap/snap0"]) - self.mount_a.run_shell(["mkdir", "d1/.snap/snap0"]) - self.mount_a.run_shell(["mkdir", "d2/.snap/snap0"]) + snap_name = "snap0" + self.mount_a.run_shell(["mkdir", f"d0/.snap/{snap_name}"]) + self.mount_a.run_shell(["mkdir", f"d1/.snap/{snap_name}"]) + self.mount_a.run_shell(["mkdir", f"d2/.snap/{snap_name}"]) - time.sleep(10) log.debug('checking snap in progress') - self.check_peer_snap_in_progress(self.primary_fs_name, self.primary_fs_id, - "client.mirror_remote@ceph", '/d0', 'snap0') - self.check_peer_snap_in_progress(self.primary_fs_name, self.primary_fs_id, - "client.mirror_remote@ceph", '/d1', 'snap0') - self.check_peer_snap_in_progress(self.primary_fs_name, self.primary_fs_id, - "client.mirror_remote@ceph", '/d2', 'snap0') + peer_spec = "client.mirror_remote@ceph" + peer_uuid = self.get_peer_uuid(peer_spec) + with safe_while(sleep=3, tries=100, action=f'wait for status: {peer_spec}') as proceed: + while proceed(): + res = self.mirror_daemon_command(f'peer status for fs: {self.primary_fs_name}', + 'fs', 'mirror', 'peer', 'status', + f'{self.primary_fs_name}@{self.primary_fs_id}', + peer_uuid) + if ('syncing' == res["/d0"]['state'] and 'syncing' == res["/d1"]['state'] and \ + 'syncing' == res["/d2"]['state']): + break - log.debug('removing directories 1') + log.debug('removing directory 1') self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d0') - log.debug('removing directories 2') + log.debug('removing directory 2') self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d1') - log.debug('removing directories 3') + log.debug('removing directory 3') self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d2') + # Wait a while for the sync backoff + time.sleep(500) + log.debug('removing snapshots') - self.mount_a.run_shell(["rmdir", "d0/.snap/snap0"]) - self.mount_a.run_shell(["rmdir", "d1/.snap/snap0"]) - self.mount_a.run_shell(["rmdir", "d2/.snap/snap0"]) + self.mount_a.run_shell(["rmdir", f"d0/.snap/{snap_name}"]) + self.mount_a.run_shell(["rmdir", f"d1/.snap/{snap_name}"]) + self.mount_a.run_shell(["rmdir", f"d2/.snap/{snap_name}"]) for i in range(4): filename = f'file.{i}' @@ -1427,26 +1435,27 @@ class TestMirroring(CephFSTestCase): self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d2') log.debug('creating new snapshots...') - self.mount_a.run_shell(["mkdir", "d0/.snap/snap0"]) - self.mount_a.run_shell(["mkdir", "d1/.snap/snap0"]) - self.mount_a.run_shell(["mkdir", "d2/.snap/snap0"]) + self.mount_a.run_shell(["mkdir", f"d0/.snap/{snap_name}"]) + self.mount_a.run_shell(["mkdir", f"d1/.snap/{snap_name}"]) + self.mount_a.run_shell(["mkdir", f"d2/.snap/{snap_name}"]) + + # Wait for the threads to finish + time.sleep(500) - time.sleep(60) self.check_peer_status(self.primary_fs_name, self.primary_fs_id, - "client.mirror_remote@ceph", '/d0', 'snap0', 1) - self.verify_snapshot('d0', 'snap0') + "client.mirror_remote@ceph", '/d0', f'{snap_name}', 1) + self.verify_snapshot('d0', f'{snap_name}') self.check_peer_status(self.primary_fs_name, self.primary_fs_id, - "client.mirror_remote@ceph", '/d1', 'snap0', 1) - self.verify_snapshot('d1', 'snap0') + "client.mirror_remote@ceph", '/d1', f'{snap_name}', 1) + self.verify_snapshot('d1', f'{snap_name}') self.check_peer_status(self.primary_fs_name, self.primary_fs_id, - "client.mirror_remote@ceph", '/d2', 'snap0', 1) - self.verify_snapshot('d2', 'snap0') + "client.mirror_remote@ceph", '/d2', f'{snap_name}', 1) + self.verify_snapshot('d2', f'{snap_name}') res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump') vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0] self.assertGreater(vafter["counters"]["snaps_synced"], vbefore["counters"]["snaps_synced"]) - self.assertGreater(vafter["counters"]["snaps_deleted"], vbefore["counters"]["snaps_deleted"]) self.disable_mirroring(self.primary_fs_name, self.primary_fs_id) -- 2.39.5