From e4dd0e41a3a01cb080656426031aa343c0ceefe6 Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Wed, 14 Feb 2024 13:18:23 -0500 Subject: [PATCH] qa: verify labelled replication perf metrics Signed-off-by: Venky Shankar (cherry picked from commit f29dd57cd0a1f609f54d62f97025ec2d67979472) Conflicts: qa/tasks/cephfs/test_mirroring.py test_mirroring.py does not use run_ceph_cmd() in reef, so, adjust for that. --- qa/tasks/cephfs/test_mirroring.py | 83 ++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/qa/tasks/cephfs/test_mirroring.py b/qa/tasks/cephfs/test_mirroring.py index 1ce5cdca8a7c8..7324dde1c6c13 100644 --- a/qa/tasks/cephfs/test_mirroring.py +++ b/qa/tasks/cephfs/test_mirroring.py @@ -21,6 +21,10 @@ class TestMirroring(CephFSTestCase): MODULE_NAME = "mirroring" + PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR = "cephfs_mirror" + PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS = "cephfs_mirror_mirrored_filesystems" + PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER = "cephfs_mirror_peers" + def setUp(self): super(TestMirroring, self).setUp() self.primary_fs_name = self.fs.name @@ -40,6 +44,9 @@ class TestMirroring(CephFSTestCase): self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "module", "disable", TestMirroring.MODULE_NAME) def enable_mirroring(self, fs_name, fs_id): + res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump') + vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR][0] + self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "enable", fs_name) time.sleep(10) # verify via asok @@ -48,7 +55,19 @@ class TestMirroring(CephFSTestCase): self.assertTrue(res['peers'] == {}) self.assertTrue(res['snap_dirs']['dir_count'] == 0) + # verify labelled perf counter + res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump') + self.assertEqual(res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0]["labels"]["filesystem"], + fs_name) + vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR][0] + + self.assertGreater(vafter["counters"]["mirrored_filesystems"], + vbefore["counters"]["mirrored_filesystems"]) + def disable_mirroring(self, fs_name, fs_id): + res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump') + vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR][0] + self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "disable", fs_name) time.sleep(10) # verify via asok @@ -60,6 +79,13 @@ class TestMirroring(CephFSTestCase): else: raise RuntimeError('expected admin socket to be unavailable') + # verify labelled perf counter + res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump') + vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR][0] + + self.assertLess(vafter["counters"]["mirrored_filesystems"], + vbefore["counters"]["mirrored_filesystems"]) + def verify_peer_added(self, fs_name, fs_id, peer_spec, remote_fs_name=None): # verify via asok res = self.mirror_daemon_command(f'mirror status for fs: {fs_name}', @@ -74,7 +100,11 @@ class TestMirroring(CephFSTestCase): else: self.assertTrue(self.fs_name == res['peers'][peer_uuid]['remote']['fs_name']) - def peer_add(self, fs_name, fs_id, peer_spec, remote_fs_name=None): + def peer_add(self, fs_name, fs_id, peer_spec, remote_fs_name=None, check_perf_counter=True): + if check_perf_counter: + res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump') + vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0] + if remote_fs_name: self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "peer_add", fs_name, peer_spec, remote_fs_name) else: @@ -82,7 +112,15 @@ class TestMirroring(CephFSTestCase): time.sleep(10) self.verify_peer_added(fs_name, fs_id, peer_spec, remote_fs_name) + if check_perf_counter: + res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump') + vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0] + self.assertGreater(vafter["counters"]["mirroring_peers"], vbefore["counters"]["mirroring_peers"]) + def peer_remove(self, fs_name, fs_id, peer_spec): + res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump') + vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0] + peer_uuid = self.get_peer_uuid(peer_spec) self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "peer_remove", fs_name, peer_uuid) time.sleep(10) @@ -91,6 +129,11 @@ class TestMirroring(CephFSTestCase): 'fs', 'mirror', 'status', f'{fs_name}@{fs_id}') self.assertTrue(res['peers'] == {} and res['snap_dirs']['dir_count'] == 0) + res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump') + vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0] + + self.assertLess(vafter["counters"]["mirroring_peers"], vbefore["counters"]["mirroring_peers"]) + def bootstrap_peer(self, fs_name, client_name, site_name): outj = json.loads(self.mgr_cluster.mon_manager.raw_cluster_cmd( "fs", "snapshot", "mirror", "peer_bootstrap", "create", fs_name, client_name, site_name)) @@ -100,7 +143,11 @@ class TestMirroring(CephFSTestCase): self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "peer_bootstrap", "import", fs_name, token) - def add_directory(self, fs_name, fs_id, dir_name): + def add_directory(self, fs_name, fs_id, dir_name, check_perf_counter=True): + if check_perf_counter: + res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump') + vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0] + # get initial dir count res = self.mirror_daemon_command(f'mirror status for fs: {fs_name}', 'fs', 'mirror', 'status', f'{fs_name}@{fs_id}') @@ -117,7 +164,14 @@ class TestMirroring(CephFSTestCase): log.debug(f'new dir_count={new_dir_count}') self.assertTrue(new_dir_count > dir_count) + if check_perf_counter: + res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump') + vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0] + self.assertGreater(vafter["counters"]["directory_count"], vbefore["counters"]["directory_count"]) + def remove_directory(self, fs_name, fs_id, dir_name): + res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump') + vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0] # get initial dir count res = self.mirror_daemon_command(f'mirror status for fs: {fs_name}', 'fs', 'mirror', 'status', f'{fs_name}@{fs_id}') @@ -134,6 +188,11 @@ class TestMirroring(CephFSTestCase): log.debug(f'new dir_count={new_dir_count}') self.assertTrue(new_dir_count < dir_count) + res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump') + vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0] + + self.assertLess(vafter["counters"]["directory_count"], vbefore["counters"]["directory_count"]) + def check_peer_status(self, fs_name, fs_id, peer_spec, dir_name, expected_snap_name, expected_snap_count): peer_uuid = self.get_peer_uuid(peer_spec) @@ -267,7 +326,7 @@ class TestMirroring(CephFSTestCase): self.enable_mirroring(self.primary_fs_name, self.primary_fs_id) try: - self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph") + self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", check_perf_counter=False) except CommandFailedError as ce: if ce.exitstatus != errno.EINVAL: raise RuntimeError('invalid errno when adding a matching remote peer') @@ -281,7 +340,7 @@ class TestMirroring(CephFSTestCase): # and explicitly specifying the spec (via filesystem name) should fail too try: - self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.primary_fs_name) + self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.primary_fs_name, check_perf_counter=False) except CommandFailedError as ce: if ce.exitstatus != errno.EINVAL: raise RuntimeError('invalid errno when adding a matching remote peer') @@ -302,7 +361,7 @@ class TestMirroring(CephFSTestCase): self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name) # adding the same peer should be idempotent - self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name) + self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name, check_perf_counter=False) # remove peer self.peer_remove(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph") @@ -312,7 +371,7 @@ class TestMirroring(CephFSTestCase): def test_peer_commands_with_mirroring_disabled(self): # try adding peer when mirroring is not enabled try: - self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name) + self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name, check_perf_counter=False) except CommandFailedError as ce: if ce.exitstatus != errno.EINVAL: raise RuntimeError(-errno.EINVAL, 'incorrect error code when adding a peer') @@ -331,7 +390,7 @@ class TestMirroring(CephFSTestCase): def test_add_directory_with_mirroring_disabled(self): # try adding a directory when mirroring is not enabled try: - self.add_directory(self.primary_fs_name, self.primary_fs_id, "/d1") + self.add_directory(self.primary_fs_name, self.primary_fs_id, "/d1", check_perf_counter=False) except CommandFailedError as ce: if ce.exitstatus != errno.EINVAL: raise RuntimeError(-errno.EINVAL, 'incorrect error code when adding a directory') @@ -343,7 +402,7 @@ class TestMirroring(CephFSTestCase): self.enable_mirroring(self.primary_fs_name, self.primary_fs_id) self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d1') try: - self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d1') + self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d1', check_perf_counter=False) except CommandFailedError as ce: if ce.exitstatus != errno.EEXIST: raise RuntimeError(-errno.EINVAL, 'incorrect error code when re-adding a directory') @@ -363,7 +422,7 @@ class TestMirroring(CephFSTestCase): def test_add_relative_directory_path(self): self.enable_mirroring(self.primary_fs_name, self.primary_fs_id) try: - self.add_directory(self.primary_fs_name, self.primary_fs_id, './d1') + self.add_directory(self.primary_fs_name, self.primary_fs_id, './d1', check_perf_counter=False) except CommandFailedError as ce: if ce.exitstatus != errno.EINVAL: raise RuntimeError(-errno.EINVAL, 'incorrect error code when adding a relative path dir') @@ -377,7 +436,7 @@ class TestMirroring(CephFSTestCase): self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d1/d2/d3') def check_add_command_failure(dir_path): try: - self.add_directory(self.primary_fs_name, self.primary_fs_id, dir_path) + self.add_directory(self.primary_fs_name, self.primary_fs_id, dir_path, check_perf_counter=False) except CommandFailedError as ce: if ce.exitstatus != errno.EEXIST: raise RuntimeError(-errno.EINVAL, 'incorrect error code when re-adding a directory') @@ -401,7 +460,7 @@ class TestMirroring(CephFSTestCase): self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d1/d2/') def check_add_command_failure(dir_path): try: - self.add_directory(self.primary_fs_name, self.primary_fs_id, dir_path) + self.add_directory(self.primary_fs_name, self.primary_fs_id, dir_path, check_perf_counter=False) except CommandFailedError as ce: if ce.exitstatus != errno.EINVAL: raise RuntimeError(-errno.EINVAL, 'incorrect error code when adding a directory') @@ -1150,7 +1209,7 @@ class TestMirroring(CephFSTestCase): # try adding the primary file system as a peer to secondary file # system try: - self.peer_add(self.secondary_fs_name, self.secondary_fs_id, "client.mirror_remote@ceph", self.primary_fs_name) + self.peer_add(self.secondary_fs_name, self.secondary_fs_id, "client.mirror_remote@ceph", self.primary_fs_name, check_perf_counter=False) except CommandFailedError as ce: if ce.exitstatus != errno.EINVAL: raise RuntimeError('invalid errno when adding a primary file system') -- 2.39.5