From 8cbdad9f9b67a2cda9b4b1e14e10b2947a72a3df Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Mon, 25 Feb 2019 11:10:01 -0800 Subject: [PATCH] qa: update testing for standby-replay Signed-off-by: Patrick Donnelly --- qa/cephfs/clusters/1-mds-1-client-coloc.yaml | 2 +- qa/cephfs/clusters/1-mds-1-client.yaml | 4 +- qa/cephfs/clusters/1-mds-2-client-coloc.yaml | 2 +- qa/cephfs/clusters/1-mds-2-client.yaml | 4 +- qa/cephfs/clusters/1-mds-3-client.yaml | 2 +- qa/cephfs/clusters/1-mds-4-client-coloc.yaml | 2 +- qa/cephfs/clusters/1-mds-4-client.yaml | 4 +- qa/cephfs/clusters/1a3s-mds-1c-client.yaml | 4 +- qa/cephfs/clusters/1a3s-mds-2c-client.yaml | 4 +- qa/cephfs/clusters/3-mds.yaml | 3 + qa/cephfs/clusters/9-mds.yaml | 3 + qa/cephfs/clusters/fixed-2-ucephfs.yaml | 2 +- qa/clusters/fixed-3-cephfs.yaml | 4 +- .../multimds/clusters/7-multimds.yaml | 6 +- .../thrash/clusters/3-mds-2-standby.yaml | 4 +- .../thrash/clusters/9-mds-3-standby.yaml | 4 +- qa/tasks/ceph.py | 11 +- qa/tasks/cephfs/filesystem.py | 27 +- qa/tasks/cephfs/test_failover.py | 449 +++++------------- qa/tasks/cephfs/test_journal_migration.py | 42 +- qa/tasks/cephfs/test_snapshots.py | 329 +++++++------ qa/tasks/mds_thrash.py | 6 +- 22 files changed, 347 insertions(+), 571 deletions(-) diff --git a/qa/cephfs/clusters/1-mds-1-client-coloc.yaml b/qa/cephfs/clusters/1-mds-1-client-coloc.yaml index abcfffec631..5fcb3d571c7 100644 --- a/qa/cephfs/clusters/1-mds-1-client-coloc.yaml +++ b/qa/cephfs/clusters/1-mds-1-client-coloc.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3, client.0] -- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7] openstack: - volumes: # attached to each instance count: 4 diff --git a/qa/cephfs/clusters/1-mds-1-client.yaml b/qa/cephfs/clusters/1-mds-1-client.yaml index e0fde446362..437a384752a 100644 --- a/qa/cephfs/clusters/1-mds-1-client.yaml +++ b/qa/cephfs/clusters/1-mds-1-client.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, mgr.y, mds.a, mds.x-s, osd.0, osd.1, osd.2, osd.3] -- [mon.b, mon.c, mgr.x, mds.y-s, osd.4, osd.5, osd.6, osd.7] +- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7] - [client.0] openstack: - volumes: # attached to each instance diff --git a/qa/cephfs/clusters/1-mds-2-client-coloc.yaml b/qa/cephfs/clusters/1-mds-2-client-coloc.yaml index 9f0f0dc390c..ac3c00d4d33 100644 --- a/qa/cephfs/clusters/1-mds-2-client-coloc.yaml +++ b/qa/cephfs/clusters/1-mds-2-client-coloc.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3, client.0] -- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7, client.1] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7, client.1] openstack: - volumes: # attached to each instance count: 4 diff --git a/qa/cephfs/clusters/1-mds-2-client.yaml b/qa/cephfs/clusters/1-mds-2-client.yaml index 83b9372de3b..d78bc7accee 100644 --- a/qa/cephfs/clusters/1-mds-2-client.yaml +++ b/qa/cephfs/clusters/1-mds-2-client.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, mgr.y, mds.a, mds.x-s, osd.0, osd.1, osd.2, osd.3] -- [mon.b, mon.c, mgr.x, mds.y-s, osd.4, osd.5, osd.6, osd.7] +- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7] - [client.0] - [client.1] openstack: diff --git a/qa/cephfs/clusters/1-mds-3-client.yaml b/qa/cephfs/clusters/1-mds-3-client.yaml index 02e6d6dc678..b78d42209e9 100644 --- a/qa/cephfs/clusters/1-mds-3-client.yaml +++ b/qa/cephfs/clusters/1-mds-3-client.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3] -- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7] - [client.0] - [client.1] - [client.2] diff --git a/qa/cephfs/clusters/1-mds-4-client-coloc.yaml b/qa/cephfs/clusters/1-mds-4-client-coloc.yaml index 6ff916c4eb8..dca03d76228 100644 --- a/qa/cephfs/clusters/1-mds-4-client-coloc.yaml +++ b/qa/cephfs/clusters/1-mds-4-client-coloc.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3, client.0, client.1] -- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7, client.2, client.3] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7, client.2, client.3] openstack: - volumes: # attached to each instance count: 4 diff --git a/qa/cephfs/clusters/1-mds-4-client.yaml b/qa/cephfs/clusters/1-mds-4-client.yaml index a451c81c23a..70dd6813ab1 100644 --- a/qa/cephfs/clusters/1-mds-4-client.yaml +++ b/qa/cephfs/clusters/1-mds-4-client.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, mgr.y, mds.a, mds.x-s, osd.0, osd.1, osd.2, osd.3] -- [mon.b, mon.c, mgr.x, mds.y-s, osd.4, osd.5, osd.6, osd.7] +- [mon.a, mgr.y, mds.a, mds.b, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mon.c, mgr.x, mds.c, osd.4, osd.5, osd.6, osd.7] - [client.0] - [client.1] - [client.2] diff --git a/qa/cephfs/clusters/1a3s-mds-1c-client.yaml b/qa/cephfs/clusters/1a3s-mds-1c-client.yaml index 0b479ed716a..2b36c9d8239 100644 --- a/qa/cephfs/clusters/1a3s-mds-1c-client.yaml +++ b/qa/cephfs/clusters/1a3s-mds-1c-client.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, mgr.y, mds.a, mds.w-s, osd.0, osd.1, osd.2, osd.3, client.0] -- [mon.b, mon.c, mgr.x, mds.x-s, mds.y-s, osd.4, osd.5, osd.6, osd.7] +- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3, client.0] +- [mon.b, mon.c, mgr.x, mds.b, mds.d, osd.4, osd.5, osd.6, osd.7] openstack: - volumes: # attached to each instance count: 4 diff --git a/qa/cephfs/clusters/1a3s-mds-2c-client.yaml b/qa/cephfs/clusters/1a3s-mds-2c-client.yaml index 01388ae541f..4be15ea161b 100644 --- a/qa/cephfs/clusters/1a3s-mds-2c-client.yaml +++ b/qa/cephfs/clusters/1a3s-mds-2c-client.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, mgr.y, mds.a, mds.w-s, osd.0, osd.1, osd.2, osd.3, client.0] -- [mon.b, mon.c, mgr.x, mds.x-s, mds.y-s, osd.4, osd.5, osd.6, osd.7, client.1] +- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3, client.0] +- [mon.b, mon.c, mgr.x, mds.b, mds.d, osd.4, osd.5, osd.6, osd.7, client.1] openstack: - volumes: # attached to each instance count: 4 diff --git a/qa/cephfs/clusters/3-mds.yaml b/qa/cephfs/clusters/3-mds.yaml index f9fc1080865..ab517757033 100644 --- a/qa/cephfs/clusters/3-mds.yaml +++ b/qa/cephfs/clusters/3-mds.yaml @@ -2,6 +2,9 @@ roles: - [mon.a, mon.c, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3] - [mon.b, mgr.x, mds.b, mds.c, osd.4, osd.5, osd.6, osd.7] - [client.0, client.1] +overrides: + ceph: + max_mds: 3 openstack: - volumes: # attached to each instance count: 4 diff --git a/qa/cephfs/clusters/9-mds.yaml b/qa/cephfs/clusters/9-mds.yaml index 414fb2ba44d..302b1ac786e 100644 --- a/qa/cephfs/clusters/9-mds.yaml +++ b/qa/cephfs/clusters/9-mds.yaml @@ -2,6 +2,9 @@ roles: - [mon.a, mon.c, mgr.y, mds.a, mds.b, mds.c, mds.d, osd.0, osd.1, osd.2, osd.3] - [mon.b, mgr.x, mds.e, mds.f, mds.g, mds.h, mds.i, osd.4, osd.5, osd.6, osd.7] - [client.0, client.1] +overrides: + ceph: + max_mds: 9 openstack: - volumes: # attached to each instance count: 4 diff --git a/qa/cephfs/clusters/fixed-2-ucephfs.yaml b/qa/cephfs/clusters/fixed-2-ucephfs.yaml index 129aac6cea2..f55f031ae5f 100644 --- a/qa/cephfs/clusters/fixed-2-ucephfs.yaml +++ b/qa/cephfs/clusters/fixed-2-ucephfs.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3, client.0] -- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7] openstack: - volumes: # attached to each instance count: 4 diff --git a/qa/clusters/fixed-3-cephfs.yaml b/qa/clusters/fixed-3-cephfs.yaml index 2d2112f9f64..9e021b3bd69 100644 --- a/qa/clusters/fixed-3-cephfs.yaml +++ b/qa/clusters/fixed-3-cephfs.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mds.a, mgr.x, osd.0, osd.1] -- [mon.b, mds.a-s, mon.c, mgr.y, osd.2, osd.3] +- [mon.b, mds.b, mon.c, mgr.y, osd.2, osd.3] - [client.0] openstack: - volumes: # attached to each instance @@ -13,4 +13,4 @@ overrides: ceph: conf: osd: - osd shutdown pgref assert: true \ No newline at end of file + osd shutdown pgref assert: true diff --git a/qa/suites/experimental/multimds/clusters/7-multimds.yaml b/qa/suites/experimental/multimds/clusters/7-multimds.yaml index d4bb14143d3..7b2763f1950 100644 --- a/qa/suites/experimental/multimds/clusters/7-multimds.yaml +++ b/qa/suites/experimental/multimds/clusters/7-multimds.yaml @@ -1,7 +1,7 @@ roles: -- [mon.a, mgr.x, mds.a, mds.a-s] -- [mon.b, mgr.y, mds.b, mds.b-s] -- [mon.c, mgr.z, mds.c, mds.c-s] +- [mon.a, mgr.x, mds.a, mds.d] +- [mon.b, mgr.y, mds.b, mds.e] +- [mon.c, mgr.z, mds.c, mds.f] - [osd.0] - [osd.1] - [osd.2] diff --git a/qa/suites/multimds/thrash/clusters/3-mds-2-standby.yaml b/qa/suites/multimds/thrash/clusters/3-mds-2-standby.yaml index 1a415ef249e..5936fcb53fe 100644 --- a/qa/suites/multimds/thrash/clusters/3-mds-2-standby.yaml +++ b/qa/suites/multimds/thrash/clusters/3-mds-2-standby.yaml @@ -1,4 +1,4 @@ roles: -- [mon.a, mon.c, mds.a, osd.0, osd.1, osd.2, mds.d-s] -- [mon.b, mgr.x, mds.b, mds.c, osd.3, osd.4, osd.5, mds.e-s] +- [mon.a, mon.c, mds.a, mds.c, mds.e, osd.0, osd.1, osd.2] +- [mon.b, mgr.x, mds.b, mds.d, osd.3, osd.4, osd.5] - [client.0] diff --git a/qa/suites/multimds/thrash/clusters/9-mds-3-standby.yaml b/qa/suites/multimds/thrash/clusters/9-mds-3-standby.yaml index 4a8334c66a2..6775e621df9 100644 --- a/qa/suites/multimds/thrash/clusters/9-mds-3-standby.yaml +++ b/qa/suites/multimds/thrash/clusters/9-mds-3-standby.yaml @@ -1,4 +1,4 @@ roles: -- [mon.a, mon.c, mds.a, mds.b, mds.c, mds.d, osd.0, osd.1, osd.2, mds.j-s, mds.k-s] -- [mon.b, mgr.x, mds.e, mds.f, mds.g, mds.h, mds.i, osd.3, osd.4, osd.5, mds.l-s] +- [mon.a, mon.c, mds.a, mds.c, mds.e, mds.g, mds.i, mds.k, osd.0, osd.1, osd.2] +- [mon.b, mgr.x, mds.b, mds.d, mds.f, mds.h, mds.j, mds.l, osd.3, osd.4, osd.5] - [client.0] diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py index 09fc825a0d9..7f097653695 100644 --- a/qa/tasks/ceph.py +++ b/qa/tasks/ceph.py @@ -410,11 +410,9 @@ def cephfs_setup(ctx, config): fs = Filesystem(ctx, name='cephfs', create=True, ec_profile=config.get('cephfs_ec_profile', None)) - is_active_mds = lambda role: 'mds.' in role and not role.endswith('-s') and '-s-' not in role - all_roles = [item for remote_roles in mdss.remotes.values() for item in remote_roles] - num_active = len([r for r in all_roles if is_active_mds(r)]) - - fs.set_max_mds(config.get('max_mds', num_active)) + max_mds = config.get('max_mds', 1) + if max_mds > 1: + fs.set_max_mds(max_mds) yield @@ -494,9 +492,6 @@ def skeleton_config(ctx, roles, ips, mons, cluster='ceph'): if is_mds(role): name = teuthology.ceph_role(role) conf.setdefault(name, {}) - if '-s-' in name: - standby_mds = name[name.find('-s-') + 3:] - conf[name]['mds standby for name'] = standby_mds return conf def create_simple_monmap(ctx, remote, conf, mons, diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py index 513517ae81c..fa6d5edc0bf 100644 --- a/qa/tasks/cephfs/filesystem.py +++ b/qa/tasks/cephfs/filesystem.py @@ -108,7 +108,7 @@ class FSStatus(object): """ fs = self.get_fsmap(fscid) for info in fs['mdsmap']['info'].values(): - if info['rank'] >= 0: + if info['rank'] >= 0 and info['state'] != 'up:standby-replay': yield info def get_rank(self, fscid, rank): @@ -484,6 +484,9 @@ class Filesystem(MDSCluster): def set_max_mds(self, max_mds): self.set_var("max_mds", "%d" % max_mds) + def set_allow_standby_replay(self, yes): + self.set_var("allow_standby_replay", str(yes).lower()) + def set_allow_new_snaps(self, yes): self.set_var("allow_new_snaps", str(yes).lower(), '--yes-i-really-mean-it') @@ -782,11 +785,33 @@ class Filesystem(MDSCluster): status = self.getinfo() return status.get_rank(self.id, rank) + def rank_restart(self, rank=0, status=None): + name = self.get_rank(rank=rank, status=status)['name'] + self.mds_restart(mds_id=name) + + def rank_signal(self, signal, rank=0, status=None): + name = self.get_rank(rank=rank, status=status)['name'] + self.mds_signal(name, signal) + + def rank_fail(self, rank=0): + self.mon_manager.raw_cluster_cmd("mds", "fail", "{}:{}".format(self.id, rank)) + def get_ranks(self, status=None): if status is None: status = self.getinfo() return status.get_ranks(self.id) + def get_replays(self, status=None): + if status is None: + status = self.getinfo() + return status.get_replays(self.id) + + def get_replay(self, rank=0, status=None): + for replay in self.get_replays(status=status): + if replay['rank'] == rank: + return replay + return None + def get_rank_names(self, status=None): """ Return MDS daemon names of those daemons holding a rank, diff --git a/qa/tasks/cephfs/test_failover.py b/qa/tasks/cephfs/test_failover.py index 9c81927ac58..c8999fa4580 100644 --- a/qa/tasks/cephfs/test_failover.py +++ b/qa/tasks/cephfs/test_failover.py @@ -3,6 +3,7 @@ import signal import json import logging from unittest import case, SkipTest +from random import randint from cephfs_test_case import CephFSTestCase from teuthology.exceptions import CommandFailedError @@ -356,155 +357,138 @@ class TestFailover(CephFSTestCase): class TestStandbyReplay(CephFSTestCase): MDSS_REQUIRED = 4 - REQUIRE_FILESYSTEM = False - def set_standby_for(self, leader, follower, replay): - self.set_conf("mds.{0}".format(follower), "mds_standby_for_name", leader) + def _confirm_no_replay(self): + status = self.fs.status() + standby_count = len(list(status.get_standbys())) + self.assertEqual(0, len(list(self.fs.get_replays(status=status)))) + return status + + def _confirm_single_replay(self, full=True, status=None): + status = self.fs.wait_for_daemons(status=status) + ranks = sorted(self.fs.get_mds_map(status=status)['in']) + replays = list(self.fs.get_replays(status=status)) + checked_replays = set() + for rank in ranks: + has_replay = False + for replay in replays: + if replay['rank'] == rank: + self.assertFalse(has_replay) + has_replay = True + checked_replays.add(replay['gid']) + if full and not has_replay: + raise RuntimeError("rank "+str(rank)+" has no standby-replay follower") + self.assertEqual(checked_replays, set(info['gid'] for info in replays)) + return status + + def _check_replay_takeover(self, status, rank=0): + replay = self.fs.get_replay(rank=rank, status=status) + new_status = self.fs.wait_for_daemons() + new_active = self.fs.get_rank(rank=rank, status=new_status) if replay: - self.set_conf("mds.{0}".format(follower), "mds_standby_replay", "true") - - def get_info_by_name(self, mds_name): - status = self.mds_cluster.status() - info = status.get_mds(mds_name) - if info is None: - log.warn(str(status)) - raise RuntimeError("MDS '{0}' not found".format(mds_name)) + self.assertEqual(replay['gid'], new_active['gid']) else: - return info + # double check takeover came from a standby (or some new daemon via restart) + found = False + for info in status.get_standbys(): + if info['gid'] == new_active['gid']: + found = True + break + if not found: + for info in status.get_all(): + self.assertNotEqual(info['gid'], new_active['gid']) + return new_status - def test_standby_replay_unused(self): - # Pick out exactly 3 daemons to be run during test - use_daemons = sorted(self.mds_cluster.mds_ids[0:3]) - mds_a, mds_b, mds_c = use_daemons - log.info("Using MDS daemons: {0}".format(use_daemons)) + def test_standby_replay_singleton(self): + """ + That only one MDS becomes standby-replay. + """ - # B and C should both follow A, but only one will - # really get into standby replay state. - self.set_standby_for(mds_a, mds_b, True) - self.set_standby_for(mds_a, mds_c, True) + self._confirm_no_replay() + self.fs.set_allow_standby_replay(True) + time.sleep(30) + self._confirm_single_replay() - # Create FS and start A - fs_a = self.mds_cluster.newfs("alpha") - self.mds_cluster.mds_restart(mds_a) - fs_a.wait_for_daemons() - self.assertEqual(fs_a.get_active_names(), [mds_a]) + def test_standby_replay_singleton_fail(self): + """ + That failures don't violate singleton constraint. + """ - # Start B, he should go into standby replay - self.mds_cluster.mds_restart(mds_b) - self.wait_for_daemon_start([mds_b]) - info_b = self.get_info_by_name(mds_b) - self.assertEqual(info_b['state'], "up:standby-replay") - self.assertEqual(info_b['standby_for_name'], mds_a) - self.assertEqual(info_b['rank'], 0) - - # Start C, he should go into standby (*not* replay) - self.mds_cluster.mds_restart(mds_c) - self.wait_for_daemon_start([mds_c]) - info_c = self.get_info_by_name(mds_c) - self.assertEqual(info_c['state'], "up:standby") - self.assertEqual(info_c['standby_for_name'], mds_a) - self.assertEqual(info_c['rank'], -1) - - # Kill B, C should go into standby replay - self.mds_cluster.mds_stop(mds_b) - self.mds_cluster.mds_fail(mds_b) - self.wait_until_equal( - lambda: self.get_info_by_name(mds_c)['state'], - "up:standby-replay", - 60) - info_c = self.get_info_by_name(mds_c) - self.assertEqual(info_c['state'], "up:standby-replay") - self.assertEqual(info_c['standby_for_name'], mds_a) - self.assertEqual(info_c['rank'], 0) - - def test_standby_failure(self): + self._confirm_no_replay() + self.fs.set_allow_standby_replay(True) + status = self._confirm_single_replay() + + for i in range(10): + time.sleep(randint(1, 5)) + self.fs.rank_restart(status=status) + status = self._check_replay_takeover(status) + status = self._confirm_single_replay(status=status) + + for i in range(10): + time.sleep(randint(1, 5)) + self.fs.rank_fail() + status = self._check_replay_takeover(status) + status = self._confirm_single_replay(status=status) + + def test_standby_replay_singleton_fail_multimds(self): + """ + That failures don't violate singleton constraint with multiple actives. + """ + + status = self._confirm_no_replay() + new_max_mds = randint(2, len(list(status.get_standbys()))) + self.fs.set_max_mds(new_max_mds) + self.fs.wait_for_daemons() # wait for actives to come online! + self.fs.set_allow_standby_replay(True) + status = self._confirm_single_replay(full=False) + + for i in range(10): + time.sleep(randint(1, 5)) + victim = randint(0, new_max_mds-1) + self.fs.rank_restart(rank=victim, status=status) + status = self._check_replay_takeover(status, rank=victim) + status = self._confirm_single_replay(status=status, full=False) + + for i in range(10): + time.sleep(randint(1, 5)) + victim = randint(0, new_max_mds-1) + self.fs.rank_fail(rank=victim) + status = self._check_replay_takeover(status, rank=victim) + status = self._confirm_single_replay(status=status, full=False) + + def test_standby_replay_failure(self): """ That the failure of a standby-replay daemon happens cleanly and doesn't interrupt anything else. """ - # Pick out exactly 2 daemons to be run during test - use_daemons = sorted(self.mds_cluster.mds_ids[0:2]) - mds_a, mds_b = use_daemons - log.info("Using MDS daemons: {0}".format(use_daemons)) - - # Configure two pairs of MDSs that are standby for each other - self.set_standby_for(mds_a, mds_b, True) - self.set_standby_for(mds_b, mds_a, False) - # Create FS alpha and get mds_a to come up as active - fs_a = self.mds_cluster.newfs("alpha") - self.mds_cluster.mds_restart(mds_a) - fs_a.wait_for_daemons() - self.assertEqual(fs_a.get_active_names(), [mds_a]) + status = self._confirm_no_replay() + self.fs.set_max_mds(1) + self.fs.set_allow_standby_replay(True) + status = self._confirm_single_replay() - # Start the standbys - self.mds_cluster.mds_restart(mds_b) - self.wait_for_daemon_start([mds_b]) - - # See the standby come up as the correct rank - info_b = self.get_info_by_name(mds_b) - self.assertEqual(info_b['state'], "up:standby-replay") - self.assertEqual(info_b['standby_for_name'], mds_a) - self.assertEqual(info_b['rank'], 0) - - # Kill the standby - self.mds_cluster.mds_stop(mds_b) - self.mds_cluster.mds_fail(mds_b) - - # See that the standby is gone and the active remains - self.assertEqual(fs_a.get_active_names(), [mds_a]) - mds_map = fs_a.get_mds_map() - self.assertEqual(len(mds_map['info']), 1) - self.assertEqual(mds_map['failed'], []) - self.assertEqual(mds_map['damaged'], []) - self.assertEqual(mds_map['stopped'], []) + for i in range(10): + time.sleep(randint(1, 5)) + victim = self.fs.get_replay(status=status) + self.fs.mds_restart(mds_id=victim['name']) + status = self._confirm_single_replay(status=status) def test_rank_stopped(self): """ That when a rank is STOPPED, standby replays for that rank get torn down """ - # Pick out exactly 2 daemons to be run during test - use_daemons = sorted(self.mds_cluster.mds_ids[0:4]) - mds_a, mds_b, mds_a_s, mds_b_s = use_daemons - log.info("Using MDS daemons: {0}".format(use_daemons)) - # a and b both get a standby - self.set_standby_for(mds_a, mds_a_s, True) - self.set_standby_for(mds_b, mds_b_s, True) - - # Create FS alpha and get mds_a to come up as active - fs_a = self.mds_cluster.newfs("alpha") - fs_a.set_max_mds(2) + status = self._confirm_no_replay() + standby_count = len(list(status.get_standbys())) + self.fs.set_max_mds(2) + self.fs.set_allow_standby_replay(True) + status = self._confirm_single_replay() - self.mds_cluster.mds_restart(mds_a) - self.wait_until_equal(lambda: fs_a.get_active_names(), [mds_a], 30) - self.mds_cluster.mds_restart(mds_b) - fs_a.wait_for_daemons() - self.assertEqual(sorted(fs_a.get_active_names()), [mds_a, mds_b]) - - # Start the standbys - self.mds_cluster.mds_restart(mds_b_s) - self.wait_for_daemon_start([mds_b_s]) - self.mds_cluster.mds_restart(mds_a_s) - self.wait_for_daemon_start([mds_a_s]) - info_b_s = self.get_info_by_name(mds_b_s) - self.assertEqual(info_b_s['state'], "up:standby-replay") - info_a_s = self.get_info_by_name(mds_a_s) - self.assertEqual(info_a_s['state'], "up:standby-replay") - - # Shrink the cluster - fs_a.set_max_mds(1) - self.wait_until_equal( - lambda: fs_a.get_active_names(), [mds_a], - 60 - ) + self.fs.set_max_mds(1) # stop rank 1 - # Both 'b' and 'b_s' should go back to being standbys - self.wait_until_equal( - lambda: self.mds_cluster.get_standby_daemons(), {mds_b, mds_b_s}, - 60 - ) + status = self._confirm_single_replay() + self.assertTrue(standby_count, len(list(status.get_standbys()))) class TestMultiFilesystems(CephFSTestCase): @@ -648,214 +632,3 @@ class TestMultiFilesystems(CephFSTestCase): fs_a.set_max_mds(3) self.wait_until_equal(lambda: len(fs_a.get_active_names()), 3, 60, reject_fn=lambda v: v > 3 or v < 2) - - def test_standby_for_name(self): - # Pick out exactly 4 daemons to be run during test - use_daemons = sorted(self.mds_cluster.mds_ids[0:4]) - mds_a, mds_b, mds_c, mds_d = use_daemons - log.info("Using MDS daemons: {0}".format(use_daemons)) - - def set_standby_for(leader, follower, replay): - self.set_conf("mds.{0}".format(follower), "mds_standby_for_name", leader) - if replay: - self.set_conf("mds.{0}".format(follower), "mds_standby_replay", "true") - - # Configure two pairs of MDSs that are standby for each other - set_standby_for(mds_a, mds_b, True) - set_standby_for(mds_b, mds_a, False) - set_standby_for(mds_c, mds_d, True) - set_standby_for(mds_d, mds_c, False) - - # Create FS alpha and get mds_a to come up as active - fs_a = self.mds_cluster.newfs("alpha") - self.mds_cluster.mds_restart(mds_a) - fs_a.wait_for_daemons() - self.assertEqual(fs_a.get_active_names(), [mds_a]) - - # Create FS bravo and get mds_c to come up as active - fs_b = self.mds_cluster.newfs("bravo") - self.mds_cluster.mds_restart(mds_c) - fs_b.wait_for_daemons() - self.assertEqual(fs_b.get_active_names(), [mds_c]) - - # Start the standbys - self.mds_cluster.mds_restart(mds_b) - self.mds_cluster.mds_restart(mds_d) - self.wait_for_daemon_start([mds_b, mds_d]) - - def get_info_by_name(fs, mds_name): - mds_map = fs.get_mds_map() - for gid_str, info in mds_map['info'].items(): - if info['name'] == mds_name: - return info - - log.warn(json.dumps(mds_map, indent=2)) - raise RuntimeError("MDS '{0}' not found in filesystem MDSMap".format(mds_name)) - - # See both standbys come up as standby replay for the correct ranks - # mds_b should be in filesystem alpha following mds_a - info_b = get_info_by_name(fs_a, mds_b) - self.assertEqual(info_b['state'], "up:standby-replay") - self.assertEqual(info_b['standby_for_name'], mds_a) - self.assertEqual(info_b['rank'], 0) - # mds_d should be in filesystem alpha following mds_c - info_d = get_info_by_name(fs_b, mds_d) - self.assertEqual(info_d['state'], "up:standby-replay") - self.assertEqual(info_d['standby_for_name'], mds_c) - self.assertEqual(info_d['rank'], 0) - - # Kill both active daemons - self.mds_cluster.mds_stop(mds_a) - self.mds_cluster.mds_fail(mds_a) - self.mds_cluster.mds_stop(mds_c) - self.mds_cluster.mds_fail(mds_c) - - # Wait for standbys to take over - fs_a.wait_for_daemons() - self.assertEqual(fs_a.get_active_names(), [mds_b]) - fs_b.wait_for_daemons() - self.assertEqual(fs_b.get_active_names(), [mds_d]) - - # Start the original active daemons up again - self.mds_cluster.mds_restart(mds_a) - self.mds_cluster.mds_restart(mds_c) - self.wait_for_daemon_start([mds_a, mds_c]) - - self.assertEqual(set(self.mds_cluster.get_standby_daemons()), - {mds_a, mds_c}) - - def test_standby_for_rank(self): - use_daemons = sorted(self.mds_cluster.mds_ids[0:4]) - mds_a, mds_b, mds_c, mds_d = use_daemons - log.info("Using MDS daemons: {0}".format(use_daemons)) - - def set_standby_for(leader_rank, leader_fs, follower_id): - self.set_conf("mds.{0}".format(follower_id), - "mds_standby_for_rank", leader_rank) - - fscid = leader_fs.get_namespace_id() - self.set_conf("mds.{0}".format(follower_id), - "mds_standby_for_fscid", fscid) - - fs_a = self.mds_cluster.newfs("alpha") - fs_b = self.mds_cluster.newfs("bravo") - set_standby_for(0, fs_a, mds_a) - set_standby_for(0, fs_a, mds_b) - set_standby_for(0, fs_b, mds_c) - set_standby_for(0, fs_b, mds_d) - - self.mds_cluster.mds_restart(mds_a) - fs_a.wait_for_daemons() - self.assertEqual(fs_a.get_active_names(), [mds_a]) - - self.mds_cluster.mds_restart(mds_c) - fs_b.wait_for_daemons() - self.assertEqual(fs_b.get_active_names(), [mds_c]) - - self.mds_cluster.mds_restart(mds_b) - self.mds_cluster.mds_restart(mds_d) - self.wait_for_daemon_start([mds_b, mds_d]) - - self.mds_cluster.mds_stop(mds_a) - self.mds_cluster.mds_fail(mds_a) - self.mds_cluster.mds_stop(mds_c) - self.mds_cluster.mds_fail(mds_c) - - fs_a.wait_for_daemons() - self.assertEqual(fs_a.get_active_names(), [mds_b]) - fs_b.wait_for_daemons() - self.assertEqual(fs_b.get_active_names(), [mds_d]) - - def test_standby_for_fscid(self): - """ - That I can set a standby FSCID with no rank, and the result is - that daemons join any rank for that filesystem. - """ - use_daemons = sorted(self.mds_cluster.mds_ids[0:4]) - mds_a, mds_b, mds_c, mds_d = use_daemons - - log.info("Using MDS daemons: {0}".format(use_daemons)) - - def set_standby_for(leader_fs, follower_id): - fscid = leader_fs.get_namespace_id() - self.set_conf("mds.{0}".format(follower_id), - "mds_standby_for_fscid", fscid) - - # Create two filesystems which should have two ranks each - fs_a = self.mds_cluster.newfs("alpha") - - fs_b = self.mds_cluster.newfs("bravo") - - fs_a.set_max_mds(2) - fs_b.set_max_mds(2) - - # Set all the daemons to have a FSCID assignment but no other - # standby preferences. - set_standby_for(fs_a, mds_a) - set_standby_for(fs_a, mds_b) - set_standby_for(fs_b, mds_c) - set_standby_for(fs_b, mds_d) - - # Now when we start all daemons at once, they should fall into - # ranks in the right filesystem - self.mds_cluster.mds_restart(mds_a) - self.mds_cluster.mds_restart(mds_b) - self.mds_cluster.mds_restart(mds_c) - self.mds_cluster.mds_restart(mds_d) - self.wait_for_daemon_start([mds_a, mds_b, mds_c, mds_d]) - fs_a.wait_for_daemons() - fs_b.wait_for_daemons() - self.assertEqual(set(fs_a.get_active_names()), {mds_a, mds_b}) - self.assertEqual(set(fs_b.get_active_names()), {mds_c, mds_d}) - - def test_standby_for_invalid_fscid(self): - """ - That an invalid standby_fscid does not cause a mon crash - """ - use_daemons = sorted(self.mds_cluster.mds_ids[0:3]) - mds_a, mds_b, mds_c = use_daemons - log.info("Using MDS daemons: {0}".format(use_daemons)) - - def set_standby_for_rank(leader_rank, follower_id): - self.set_conf("mds.{0}".format(follower_id), - "mds_standby_for_rank", leader_rank) - - # Create one fs - fs_a = self.mds_cluster.newfs("cephfs") - - # Get configured mons in the cluster, so we can see if any - # crashed later. - configured_mons = fs_a.mon_manager.get_mon_quorum() - - # Set all the daemons to have a rank assignment but no other - # standby preferences. - set_standby_for_rank(0, mds_a) - set_standby_for_rank(0, mds_b) - - # Set third daemon to have invalid fscid assignment and no other - # standby preferences - invalid_fscid = 123 - self.set_conf("mds.{0}".format(mds_c), "mds_standby_for_fscid", invalid_fscid) - - #Restart all the daemons to make the standby preference applied - self.mds_cluster.mds_restart(mds_a) - self.mds_cluster.mds_restart(mds_b) - self.mds_cluster.mds_restart(mds_c) - self.wait_for_daemon_start([mds_a, mds_b, mds_c]) - - #Stop active mds daemon service of fs - if (fs_a.get_active_names(), [mds_a]): - self.mds_cluster.mds_stop(mds_a) - self.mds_cluster.mds_fail(mds_a) - fs_a.wait_for_daemons() - else: - self.mds_cluster.mds_stop(mds_b) - self.mds_cluster.mds_fail(mds_b) - fs_a.wait_for_daemons() - - #Get active mons from cluster - active_mons = fs_a.mon_manager.get_mon_quorum() - - #Check for active quorum mon status and configured mon status - self.assertEqual(active_mons, configured_mons, - "Not all mons are in quorum Invalid standby invalid fscid test failed!") diff --git a/qa/tasks/cephfs/test_journal_migration.py b/qa/tasks/cephfs/test_journal_migration.py index 5f956be93aa..bd431dd3d97 100644 --- a/qa/tasks/cephfs/test_journal_migration.py +++ b/qa/tasks/cephfs/test_journal_migration.py @@ -15,35 +15,22 @@ class TestJournalMigration(CephFSTestCase): old_journal_version = JOURNAL_FORMAT_LEGACY new_journal_version = JOURNAL_FORMAT_RESILIENT - # Pick out two daemons to use - mds_a, mds_b = sorted(self.mds_cluster.mds_ids[0:2]) - self.mount_a.umount_wait() self.fs.mds_stop() - # Enable standby replay, to cover the bug case #8811 where - # a standby replay might mistakenly end up trying to rewrite - # the journal at the same time as an active daemon. - self.fs.set_ceph_conf('mds', 'mds standby replay', "true") - self.fs.set_ceph_conf('mds', 'mds standby for rank', "0") - # Create a filesystem using the older journal format. self.fs.set_ceph_conf('mds', 'mds journal format', old_journal_version) + self.fs.mds_restart() self.fs.recreate() - self.fs.mds_restart(mds_id=mds_a) - self.fs.wait_for_daemons() - self.assertEqual(self.fs.get_active_names(), [mds_a]) - def replay_names(): - return [s['name'] - for s in self.fs.status().get_replays(fscid = self.fs.id)] + # Enable standby replay, to cover the bug case #8811 where + # a standby replay might mistakenly end up trying to rewrite + # the journal at the same time as an active daemon. + self.fs.set_allow_standby_replay(True) - # Start the standby and wait for it to come up - self.fs.mds_restart(mds_id=mds_b) - self.wait_until_equal( - replay_names, - [mds_b], - timeout = 30) + status = self.fs.wait_for_daemons() + + self.assertTrue(self.fs.get_replay(status=status) is not None) # Do some client work so that the log is populated with something. with self.mount_a.mounted(): @@ -63,11 +50,10 @@ class TestJournalMigration(CephFSTestCase): self.fs.set_ceph_conf('mds', 'mds journal format', new_journal_version) # Restart the MDS. - self.fs.mds_fail_restart(mds_id=mds_a) - self.fs.mds_fail_restart(mds_id=mds_b) + self.fs.mds_fail_restart() # This ensures that all daemons come up into a valid state - self.fs.wait_for_daemons() + status = self.fs.wait_for_daemons() # Check that files created in the initial client workload are still visible # in a client mount. @@ -112,8 +98,6 @@ class TestJournalMigration(CephFSTestCase): }) # Check that both an active and a standby replay are still up - self.assertEqual(len(replay_names()), 1) - self.assertEqual(len(self.fs.get_active_names()), 1) - self.assertTrue(self.mds_cluster.mds_daemons[mds_a].running()) - self.assertTrue(self.mds_cluster.mds_daemons[mds_b].running()) - + status = self.fs.status() + self.assertEqual(len(list(self.fs.get_replays(status=status))), 1) + self.assertEqual(len(list(self.fs.get_ranks(status=status))), 1) diff --git a/qa/tasks/cephfs/test_snapshots.py b/qa/tasks/cephfs/test_snapshots.py index db2b8f74302..7c45ed877ee 100644 --- a/qa/tasks/cephfs/test_snapshots.py +++ b/qa/tasks/cephfs/test_snapshots.py @@ -1,4 +1,5 @@ import logging +import signal import time from textwrap import dedent from tasks.cephfs.fuse_mount import FuseMount @@ -13,36 +14,30 @@ MDS_RESTART_GRACE = 60 class TestSnapshots(CephFSTestCase): MDSS_REQUIRED = 3 - def _check_subtree(self, status, rank, path): - got_subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, rank)['name']) + def _check_subtree(self, rank, path, status=None): + got_subtrees = self.fs.rank_asok(["get", "subtrees"], rank=rank, status=status) for s in got_subtrees: if s['dir']['path'] == path and s['auth_first'] == rank: return True return False - def _stop_standby_mds(self): - for i in self.fs.get_standby_daemons(): - self.fs.mds_stop(i) - self.fs.mds_fail(i) - self.wait_until_equal(lambda: len(self.fs.get_standby_daemons()), expect_val=0, timeout=30) + def _get_snapclient_dump(self, rank=0, status=None): + return self.fs.rank_asok(["dump", "snaps"], rank=rank, status=status) - def _get_snapclient_dump(self, rank_id): - return self.fs.mds_asok(["dump", "snaps"], rank_id) + def _get_snapserver_dump(self, rank=0, status=None): + return self.fs.rank_asok(["dump", "snaps", "--server"], rank=rank, status=status) - def _get_snapserver_dump(self, rank0_id): - return self.fs.mds_asok(["dump", "snaps", "--server"], rank0_id) + def _get_last_created_snap(self, rank=0, status=None): + return int(self._get_snapserver_dump(rank,status=status)["last_created"]) - def _get_last_created_snap(self, rank0_id): - return int(self._get_snapserver_dump(rank0_id)["last_created"]) + def _get_last_destroyed_snap(self, rank=0, status=None): + return int(self._get_snapserver_dump(rank,status=status)["last_destroyed"]) - def _get_last_destroyed_snap(self, rank0_id): - return int(self._get_snapserver_dump(rank0_id)["last_destroyed"]) + def _get_pending_snap_update(self, rank=0, status=None): + return self._get_snapserver_dump(rank,status=status)["pending_update"] - def _get_pending_snap_update(self, rank0_id): - return self._get_snapserver_dump(rank0_id)["pending_update"] - - def _get_pending_snap_destroy(self, rank0_id): - return self._get_snapserver_dump(rank0_id)["pending_destroy"] + def _get_pending_snap_destroy(self, rank=0, status=None): + return self._get_snapserver_dump(rank,status=status)["pending_destroy"] def test_kill_mdstable(self): """ @@ -53,24 +48,16 @@ class TestSnapshots(CephFSTestCase): self.fs.set_allow_new_snaps(True); self.fs.set_max_mds(2) - self.fs.wait_for_daemons() + status = self.fs.wait_for_daemons() grace = float(self.fs.get_config("mds_beacon_grace", service_type="mon")) - self._stop_standby_mds() - - status = self.fs.status() - rank0_id=status.get_rank(self.fs.id, 0)['name'] - rank1_id=status.get_rank(self.fs.id, 1)['name'] - self.set_conf("mds.{0}".format(rank0_id), "mds standby for rank", 0) - self.set_conf("mds.{0}".format(rank1_id), "mds standby for rank", 1) - # setup subtrees self.mount_a.run_shell(["mkdir", "-p", "d1/dir"]) self.mount_a.setfattr("d1", "ceph.dir.pin", "1") - self.wait_until_true(lambda: self._check_subtree(status, 1, '/d1'), timeout=30) + self.wait_until_true(lambda: self._check_subtree(1, '/d1', status=status), timeout=30) - last_created = self._get_last_created_snap(rank0_id) + last_created = self._get_last_created_snap(rank=0,status=status) # mds_kill_mdstable_at: # 1: MDSTableServer::handle_prepare @@ -79,18 +66,23 @@ class TestSnapshots(CephFSTestCase): # 6: MDSTableServer::_commit_logged for i in [1,2,5,6]: log.info("testing snapserver mds_kill_mdstable_at={0}".format(i)) - self.fs.mds_asok(['config', 'set', "mds_kill_mdstable_at", "{0}".format(i)], rank0_id) + + status = self.fs.status() + rank0 = self.fs.get_rank(rank=0, status=status) + self.fs.rank_freeze(True, rank=0) + self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "{0}".format(i)], rank=0, status=status) proc = self.mount_a.run_shell(["mkdir", "d1/dir/.snap/s1{0}".format(i)], wait=False) - self.wait_until_true(lambda: "laggy_since" in self.fs.status().get_mds(rank0_id), timeout=grace*2); - self.delete_mds_coredump(rank0_id); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=0), timeout=grace*2); + self.delete_mds_coredump(rank0['name']); - self.mds_cluster.mds_fail_restart(rank0_id) - self.wait_for_daemon_start([rank0_id]) - self.fs.wait_for_state('up:active', rank=0, timeout=MDS_RESTART_GRACE) + self.fs.rank_fail(rank=0) + self.fs.mds_restart(rank0['name']) + self.wait_for_daemon_start([rank0['name']]) + status = self.fs.wait_for_daemons() proc.wait() last_created += 1 - self.wait_until_true(lambda: self._get_last_created_snap(rank0_id) == last_created, timeout=30) + self.wait_until_true(lambda: self._get_last_created_snap(rank=0) == last_created, timeout=30) self.set_conf("mds", "mds_reconnect_timeout", "5") @@ -99,39 +91,45 @@ class TestSnapshots(CephFSTestCase): # set mds_kill_mdstable_at, also kill snapclient for i in [2,5,6]: log.info("testing snapserver mds_kill_mdstable_at={0}, also kill snapclient".format(i)) - last_created = self._get_last_created_snap(rank0_id) - - self.fs.mds_asok(['config', 'set', "mds_kill_mdstable_at", "{0}".format(i)], rank0_id) + status = self.fs.status() + last_created = self._get_last_created_snap(rank=0, status=status) + + rank0 = self.fs.get_rank(rank=0, status=status) + rank1 = self.fs.get_rank(rank=1, status=status) + self.fs.rank_freeze(True, rank=0) # prevent failover... + self.fs.rank_freeze(True, rank=1) # prevent failover... + self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "{0}".format(i)], rank=0, status=status) proc = self.mount_a.run_shell(["mkdir", "d1/dir/.snap/s2{0}".format(i)], wait=False) - self.wait_until_true(lambda: "laggy_since" in self.fs.status().get_mds(rank0_id), timeout=grace*2); - self.delete_mds_coredump(rank0_id); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=0), timeout=grace*2); + self.delete_mds_coredump(rank0['name']); - self.mds_cluster.mds_stop(rank1_id) - self.mds_cluster.mds_fail(rank1_id); + self.fs.rank_signal(signal.SIGKILL, rank=1) self.mount_a.kill() self.mount_a.kill_cleanup() - self.mds_cluster.mds_fail_restart(rank0_id) - self.wait_for_daemon_start([rank0_id]) + self.fs.rank_fail(rank=0) + self.fs.mds_restart(rank0['name']) + self.wait_for_daemon_start([rank0['name']]) self.fs.wait_for_state('up:resolve', rank=0, timeout=MDS_RESTART_GRACE) if i in [2,5]: - self.assertEqual(len(self._get_pending_snap_update(rank0_id)), 1) + self.assertEqual(len(self._get_pending_snap_update(rank=0)), 1) elif i == 6: - self.assertEqual(len(self._get_pending_snap_update(rank0_id)), 0) - self.assertGreater(self._get_last_created_snap(rank0_id), last_created) + self.assertEqual(len(self._get_pending_snap_update(rank=0)), 0) + self.assertGreater(self._get_last_created_snap(rank=0), last_created) - self.mds_cluster.mds_restart(rank1_id) - self.wait_for_daemon_start([rank1_id]) + self.fs.rank_fail(rank=1) + self.fs.mds_restart(rank1['name']) + self.wait_for_daemon_start([rank1['name']]) self.fs.wait_for_state('up:active', rank=0, timeout=MDS_RESTART_GRACE) if i in [2,5]: - self.wait_until_true(lambda: len(self._get_pending_snap_update(rank0_id)) == 0, timeout=30) + self.wait_until_true(lambda: len(self._get_pending_snap_update(rank=0)) == 0, timeout=30) if i == 2: - self.assertEqual(self._get_last_created_snap(rank0_id), last_created) + self.assertEqual(self._get_last_created_snap(rank=0), last_created) else: - self.assertGreater(self._get_last_created_snap(rank0_id), last_created) + self.assertGreater(self._get_last_created_snap(rank=0), last_created) self.mount_a.mount() self.mount_a.wait_until_mounted() @@ -144,32 +142,36 @@ class TestSnapshots(CephFSTestCase): # 7: MDSTableClient::handle_request (got ack) for i in [3,4,7]: log.info("testing snapclient mds_kill_mdstable_at={0}".format(i)) - last_created = self._get_last_created_snap(rank0_id) + last_created = self._get_last_created_snap(rank=0) - self.fs.mds_asok(['config', 'set', "mds_kill_mdstable_at", "{0}".format(i)], rank1_id) + status = self.fs.status() + rank1 = self.fs.get_rank(rank=1, status=status) + self.fs.rank_freeze(True, rank=1) # prevent failover... + self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "{0}".format(i)], rank=1, status=status) proc = self.mount_a.run_shell(["mkdir", "d1/dir/.snap/s3{0}".format(i)], wait=False) - self.wait_until_true(lambda: "laggy_since" in self.fs.status().get_mds(rank1_id), timeout=grace*2); - self.delete_mds_coredump(rank1_id); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=1), timeout=grace*2); + self.delete_mds_coredump(rank1['name']); self.mount_a.kill() self.mount_a.kill_cleanup() if i in [3,4]: - self.assertEqual(len(self._get_pending_snap_update(rank0_id)), 1) + self.assertEqual(len(self._get_pending_snap_update(rank=0)), 1) elif i == 7: - self.assertEqual(len(self._get_pending_snap_update(rank0_id)), 0) - self.assertGreater(self._get_last_created_snap(rank0_id), last_created) + self.assertEqual(len(self._get_pending_snap_update(rank=0)), 0) + self.assertGreater(self._get_last_created_snap(rank=0), last_created) - self.mds_cluster.mds_fail_restart(rank1_id) - self.wait_for_daemon_start([rank1_id]) - self.fs.wait_for_state('up:active', rank=1, timeout=MDS_RESTART_GRACE) + self.fs.rank_fail(rank=1) + self.fs.mds_restart(rank1['name']) + self.wait_for_daemon_start([rank1['name']]) + status = self.fs.wait_for_daemons(timeout=MDS_RESTART_GRACE) if i in [3,4]: - self.wait_until_true(lambda: len(self._get_pending_snap_update(rank0_id)) == 0, timeout=30) + self.wait_until_true(lambda: len(self._get_pending_snap_update(rank=0)) == 0, timeout=30) if i == 3: - self.assertEqual(self._get_last_created_snap(rank0_id), last_created) + self.assertEqual(self._get_last_created_snap(rank=0), last_created) else: - self.assertGreater(self._get_last_created_snap(rank0_id), last_created) + self.assertGreater(self._get_last_created_snap(rank=0), last_created) self.mount_a.mount() self.mount_a.wait_until_mounted() @@ -180,31 +182,39 @@ class TestSnapshots(CephFSTestCase): # 3: MDSTableClient::handle_request (got agree) # 8: MDSTableServer::handle_rollback log.info("testing snapclient mds_kill_mdstable_at=3, snapserver mds_kill_mdstable_at=8") - last_created = self._get_last_created_snap(rank0_id) + last_created = self._get_last_created_snap(rank=0) - self.fs.mds_asok(['config', 'set', "mds_kill_mdstable_at", "8".format(i)], rank0_id) - self.fs.mds_asok(['config', 'set', "mds_kill_mdstable_at", "3".format(i)], rank1_id) + status = self.fs.status() + rank0 = self.fs.get_rank(rank=0, status=status) + rank1 = self.fs.get_rank(rank=1, status=status) + self.fs.rank_freeze(True, rank=0) + self.fs.rank_freeze(True, rank=1) + self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "8".format(i)], rank=0, status=status) + self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "3".format(i)], rank=1, status=status) proc = self.mount_a.run_shell(["mkdir", "d1/dir/.snap/s4".format(i)], wait=False) - self.wait_until_true(lambda: "laggy_since" in self.fs.status().get_mds(rank1_id), timeout=grace*2); - self.delete_mds_coredump(rank1_id); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=1), timeout=grace*2); + self.delete_mds_coredump(rank1['name']); self.mount_a.kill() self.mount_a.kill_cleanup() - self.assertEqual(len(self._get_pending_snap_update(rank0_id)), 1) + self.assertEqual(len(self._get_pending_snap_update(rank=0)), 1) - self.mds_cluster.mds_fail_restart(rank1_id) + self.fs.rank_fail(rank=1) + self.fs.mds_restart(rank1['name']) + self.wait_for_daemon_start([rank1['name']]) # rollback triggers assertion - self.wait_until_true(lambda: "laggy_since" in self.fs.status().get_mds(rank0_id), timeout=grace*2); - self.delete_mds_coredump(rank0_id); - - self.mds_cluster.mds_fail_restart(rank0_id) - self.wait_for_daemon_start([rank0_id]) + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=0), timeout=grace*2); + self.delete_mds_coredump(rank0['name']); + self.fs.rank_fail(rank=0) + self.fs.mds_restart(rank0['name']) + self.wait_for_daemon_start([rank0['name']]) self.fs.wait_for_state('up:active', rank=0, timeout=MDS_RESTART_GRACE) - # mds.1 should re-send rollack message - self.wait_until_true(lambda: len(self._get_pending_snap_update(rank0_id)) == 0, timeout=30) - self.assertEqual(self._get_last_created_snap(rank0_id), last_created) + + # mds.1 should re-send rollback message + self.wait_until_true(lambda: len(self._get_pending_snap_update(rank=0)) == 0, timeout=30) + self.assertEqual(self._get_last_created_snap(rank=0), last_created) self.mount_a.mount() self.mount_a.wait_until_mounted() @@ -215,92 +225,81 @@ class TestSnapshots(CephFSTestCase): """ self.fs.set_allow_new_snaps(True); self.fs.set_max_mds(3) - self.fs.wait_for_daemons() + status = self.fs.wait_for_daemons() grace = float(self.fs.get_config("mds_beacon_grace", service_type="mon")) - self._stop_standby_mds() - - status = self.fs.status() - rank0_id=status.get_rank(self.fs.id, 0)['name'] - rank1_id=status.get_rank(self.fs.id, 1)['name'] - rank2_id=status.get_rank(self.fs.id, 2)['name'] - self.set_conf("mds.{0}".format(rank0_id), "mds standby for rank", 0) - self.set_conf("mds.{0}".format(rank1_id), "mds standby for rank", 1) - self.set_conf("mds.{0}".format(rank2_id), "mds standby for rank", 2) - self.mount_a.run_shell(["mkdir", "-p", "d0/d1/dir"]) self.mount_a.run_shell(["mkdir", "-p", "d0/d2/dir"]) self.mount_a.setfattr("d0", "ceph.dir.pin", "0") self.mount_a.setfattr("d0/d1", "ceph.dir.pin", "1") self.mount_a.setfattr("d0/d2", "ceph.dir.pin", "2") - self.wait_until_true(lambda: self._check_subtree(status, 2, '/d0/d2'), timeout=30) - self.wait_until_true(lambda: self._check_subtree(status, 1, '/d0/d1'), timeout=5) - self.wait_until_true(lambda: self._check_subtree(status, 0, '/d0'), timeout=5) + self.wait_until_true(lambda: self._check_subtree(2, '/d0/d2', status=status), timeout=30) + self.wait_until_true(lambda: self._check_subtree(1, '/d0/d1', status=status), timeout=5) + self.wait_until_true(lambda: self._check_subtree(0, '/d0', status=status), timeout=5) - def _check_snapclient_cache(snaps_dump, cache_dump=None, rank_id=-1): + def _check_snapclient_cache(snaps_dump, cache_dump=None, rank=0): if cache_dump is None: - cache_dump = self._get_snapclient_dump(rank_id) + cache_dump = self._get_snapclient_dump(rank=rank) for key, value in cache_dump.iteritems(): if value != snaps_dump[key]: return False return True; # sync after mksnap - last_created = self._get_last_created_snap(rank0_id) - self.mount_a.run_shell(["mkdir", Raw("d0/d1/dir/.snap/{s1,s2}")]) - self.wait_until_true(lambda: len(self._get_pending_snap_update(rank0_id)) == 0, timeout=30) - self.assertGreater(self._get_last_created_snap(rank0_id), last_created) + last_created = self._get_last_created_snap(rank=0) + self.mount_a.run_shell(["mkdir", "d0/d1/dir/.snap/s1", "d0/d1/dir/.snap/s2"]) + self.wait_until_true(lambda: len(self._get_pending_snap_update(rank=0)) == 0, timeout=30) + self.assertGreater(self._get_last_created_snap(rank=0), last_created) - snaps_dump = self._get_snapserver_dump(rank0_id) - self.assertTrue(_check_snapclient_cache(snaps_dump, rank_id=rank0_id)); - self.assertTrue(_check_snapclient_cache(snaps_dump, rank_id=rank1_id)); - self.assertTrue(_check_snapclient_cache(snaps_dump, rank_id=rank2_id)); + snaps_dump = self._get_snapserver_dump(rank=0) + self.assertTrue(_check_snapclient_cache(snaps_dump, rank=0)); + self.assertTrue(_check_snapclient_cache(snaps_dump, rank=1)); + self.assertTrue(_check_snapclient_cache(snaps_dump, rank=2)); # sync after rmsnap - last_destroyed = self._get_last_destroyed_snap(rank0_id) + last_destroyed = self._get_last_destroyed_snap(rank=0) self.mount_a.run_shell(["rmdir", "d0/d1/dir/.snap/s1"]) - self.wait_until_true(lambda: len(self._get_pending_snap_destroy(rank0_id)) == 0, timeout=30) - self.assertGreater(self._get_last_destroyed_snap(rank0_id), last_destroyed) + self.wait_until_true(lambda: len(self._get_pending_snap_destroy(rank=0)) == 0, timeout=30) + self.assertGreater(self._get_last_destroyed_snap(rank=0), last_destroyed) - snaps_dump = self._get_snapserver_dump(rank0_id) - self.assertTrue(_check_snapclient_cache(snaps_dump, rank_id=rank0_id)); - self.assertTrue(_check_snapclient_cache(snaps_dump, rank_id=rank1_id)); - self.assertTrue(_check_snapclient_cache(snaps_dump, rank_id=rank2_id)); + snaps_dump = self._get_snapserver_dump(rank=0) + self.assertTrue(_check_snapclient_cache(snaps_dump, rank=0)); + self.assertTrue(_check_snapclient_cache(snaps_dump, rank=1)); + self.assertTrue(_check_snapclient_cache(snaps_dump, rank=2)); # sync during mds recovers - self.mds_cluster.mds_stop(rank2_id) - self.mds_cluster.mds_fail_restart(rank2_id) - self.wait_for_daemon_start([rank2_id]) - self.fs.wait_for_state('up:active', rank=2, timeout=MDS_RESTART_GRACE) - self.assertTrue(_check_snapclient_cache(snaps_dump, rank_id=rank2_id)); - - self.mds_cluster.mds_stop(rank0_id) - self.mds_cluster.mds_stop(rank1_id) - self.mds_cluster.mds_fail_restart(rank0_id) - self.mds_cluster.mds_fail_restart(rank1_id) - self.wait_for_daemon_start([rank0_id, rank1_id]) + self.fs.rank_fail(rank=2) + status = self.fs.wait_for_daemons(timeout=MDS_RESTART_GRACE) + self.assertTrue(_check_snapclient_cache(snaps_dump, rank=2)); + + self.fs.rank_fail(rank=0) + self.fs.rank_fail(rank=1) + status = self.fs.wait_for_daemons() self.fs.wait_for_state('up:active', rank=0, timeout=MDS_RESTART_GRACE) - self.assertTrue(_check_snapclient_cache(snaps_dump, rank_id=rank0_id)); - self.assertTrue(_check_snapclient_cache(snaps_dump, rank_id=rank1_id)); - self.assertTrue(_check_snapclient_cache(snaps_dump, rank_id=rank2_id)); + self.assertTrue(_check_snapclient_cache(snaps_dump, rank=0)); + self.assertTrue(_check_snapclient_cache(snaps_dump, rank=1)); + self.assertTrue(_check_snapclient_cache(snaps_dump, rank=2)); # kill at MDSTableClient::handle_notify_prep - self.fs.mds_asok(['config', 'set', "mds_kill_mdstable_at", "9"], rank2_id) + status = self.fs.status() + rank2 = self.fs.get_rank(rank=2, status=status) + self.fs.rank_freeze(True, rank=2) + self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "9"], rank=2, status=status) proc = self.mount_a.run_shell(["mkdir", "d0/d1/dir/.snap/s3"], wait=False) - self.wait_until_true(lambda: "laggy_since" in self.fs.status().get_mds(rank2_id), timeout=grace*2); - self.delete_mds_coredump(rank2_id); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=2), timeout=grace*2); + self.delete_mds_coredump(rank2['name']); # mksnap should wait for notify ack from mds.2 self.assertFalse(proc.finished); # mksnap should proceed after mds.2 fails - self.mds_cluster.mds_fail(rank2_id) + self.fs.rank_fail(rank=2) self.wait_until_true(lambda: proc.finished, timeout=30); - self.mds_cluster.mds_fail_restart(rank2_id) - self.wait_for_daemon_start([rank2_id]) - self.fs.wait_for_state('up:active', rank=2, timeout=MDS_RESTART_GRACE) + self.fs.mds_restart(rank2['name']) + self.wait_for_daemon_start([rank2['name']]) + status = self.fs.wait_for_daemons(timeout=MDS_RESTART_GRACE) self.mount_a.run_shell(["rmdir", Raw("d0/d1/dir/.snap/*")]) @@ -308,39 +307,41 @@ class TestSnapshots(CephFSTestCase): # the recovering mds should sync all mds' cache when it enters resolve stage self.set_conf("mds", "mds_reconnect_timeout", "5") for i in range(1, 4): - self.fs.mds_asok(['config', 'set', "mds_kill_mdstable_at", "4"], rank2_id) - last_created = self._get_last_created_snap(rank0_id) + status = self.fs.status() + rank2 = self.fs.get_rank(rank=2, status=status) + self.fs.rank_freeze(True, rank=2) + self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "4"], rank=2, status=status) + last_created = self._get_last_created_snap(rank=0) proc = self.mount_a.run_shell(["mkdir", "d0/d2/dir/.snap/s{0}".format(i)], wait=False) - self.wait_until_true(lambda: "laggy_since" in self.fs.status().get_mds(rank2_id), timeout=grace*2); - self.delete_mds_coredump(rank2_id); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=2), timeout=grace*2); + self.delete_mds_coredump(rank2['name']); self.mount_a.kill() self.mount_a.kill_cleanup() - self.assertEqual(len(self._get_pending_snap_update(rank0_id)), 1) + self.assertEqual(len(self._get_pending_snap_update(rank=0)), 1) if i in [2,4]: - self.mds_cluster.mds_stop(rank0_id) - self.mds_cluster.mds_fail_restart(rank0_id) + self.fs.rank_fail(rank=0) if i in [3,4]: - self.mds_cluster.mds_stop(rank1_id) - self.mds_cluster.mds_fail_restart(rank1_id) + self.fs.rank_fail(rank=1) - self.mds_cluster.mds_fail_restart(rank2_id) - self.wait_for_daemon_start([rank0_id, rank1_id, rank2_id]) + self.fs.rank_fail(rank=2) + self.fs.mds_restart(rank2['name']) + self.wait_for_daemon_start([rank2['name']]) + status = self.fs.wait_for_daemons(timeout=MDS_RESTART_GRACE) - self.fs.wait_for_state('up:active', rank=2, timeout=MDS_RESTART_GRACE) - rank0_cache = self._get_snapclient_dump(rank0_id) - rank1_cache = self._get_snapclient_dump(rank1_id) - rank2_cache = self._get_snapclient_dump(rank2_id) + rank0_cache = self._get_snapclient_dump(rank=0) + rank1_cache = self._get_snapclient_dump(rank=1) + rank2_cache = self._get_snapclient_dump(rank=2) self.assertGreater(int(rank0_cache["last_created"]), last_created) self.assertEqual(rank0_cache, rank1_cache); self.assertEqual(rank0_cache, rank2_cache); - self.wait_until_true(lambda: len(self._get_pending_snap_update(rank0_id)) == 0, timeout=30) + self.wait_until_true(lambda: len(self._get_pending_snap_update(rank=0)) == 0, timeout=30) - snaps_dump = self._get_snapserver_dump(rank0_id) + snaps_dump = self._get_snapserver_dump(rank=0) self.assertEqual(snaps_dump["last_created"], rank0_cache["last_created"]) self.assertTrue(_check_snapclient_cache(snaps_dump, cache_dump=rank0_cache)); @@ -355,15 +356,13 @@ class TestSnapshots(CephFSTestCase): """ self.fs.set_allow_new_snaps(True); self.fs.set_max_mds(2) - self.fs.wait_for_daemons() - - status = self.fs.status() + status = self.fs.wait_for_daemons() self.mount_a.run_shell(["mkdir", "-p", "d0/d1"]) self.mount_a.setfattr("d0", "ceph.dir.pin", "0") self.mount_a.setfattr("d0/d1", "ceph.dir.pin", "1") - self.wait_until_true(lambda: self._check_subtree(status, 1, '/d0/d1'), timeout=30) - self.wait_until_true(lambda: self._check_subtree(status, 0, '/d0'), timeout=5) + self.wait_until_true(lambda: self._check_subtree(1, '/d0/d1', status=status), timeout=30) + self.wait_until_true(lambda: self._check_subtree(0, '/d0', status=status), timeout=5) self.mount_a.write_test_pattern("d0/d1/file_a", 8 * 1024 * 1024) self.mount_a.run_shell(["mkdir", "d0/.snap/s1"]) @@ -379,15 +378,13 @@ class TestSnapshots(CephFSTestCase): """ self.fs.set_allow_new_snaps(True); self.fs.set_max_mds(2) - self.fs.wait_for_daemons() - - status = self.fs.status() + status = self.fs.wait_for_daemons() self.mount_a.run_shell(["mkdir", "d0", "d1"]) self.mount_a.setfattr("d0", "ceph.dir.pin", "0") self.mount_a.setfattr("d1", "ceph.dir.pin", "1") - self.wait_until_true(lambda: self._check_subtree(status, 1, '/d1'), timeout=30) - self.wait_until_true(lambda: self._check_subtree(status, 0, '/d0'), timeout=5) + self.wait_until_true(lambda: self._check_subtree(1, '/d1', status=status), timeout=30) + self.wait_until_true(lambda: self._check_subtree(0, '/d0', status=status), timeout=5) self.mount_a.run_shell(["mkdir", "d0/d3"]) self.mount_a.run_shell(["mkdir", "d0/.snap/s1"]) @@ -409,16 +406,14 @@ class TestSnapshots(CephFSTestCase): """ self.fs.set_allow_new_snaps(True); self.fs.set_max_mds(2) - self.fs.wait_for_daemons() - - status = self.fs.status() + status = self.fs.wait_for_daemons() self.mount_a.run_shell(["mkdir", "d0", "d1"]) self.mount_a.setfattr("d0", "ceph.dir.pin", "0") self.mount_a.setfattr("d1", "ceph.dir.pin", "1") - self.wait_until_true(lambda: self._check_subtree(status, 1, '/d1'), timeout=30) - self.wait_until_true(lambda: self._check_subtree(status, 0, '/d0'), timeout=5) + self.wait_until_true(lambda: self._check_subtree(1, '/d1', status=status), timeout=30) + self.wait_until_true(lambda: self._check_subtree(0, '/d0', status=status), timeout=5) self.mount_a.run_python(dedent(""" import os diff --git a/qa/tasks/mds_thrash.py b/qa/tasks/mds_thrash.py index af5fe784261..30543b56f1b 100644 --- a/qa/tasks/mds_thrash.py +++ b/qa/tasks/mds_thrash.py @@ -269,10 +269,10 @@ class MDSThrasher(Greenlet): "powercycling requested but RemoteConsole is not " "initialized. Check ipmi config.") - def revive_mds(self, mds, standby_for_rank=None): + def revive_mds(self, mds): """ Revive mds -- do an ipmpi powercycle (if indicated by the config) - and then restart (using --hot-standby if specified. + and then restart. """ if self.config.get('powercycle'): (remote,) = (self.ctx.cluster.only('mds.{m}'.format(m=mds)). @@ -283,8 +283,6 @@ class MDSThrasher(Greenlet): remote.console.power_on() self.manager.make_admin_daemon_dir(self.ctx, remote) args = [] - if standby_for_rank: - args.extend(['--hot-standby', standby_for_rank]) self.ctx.daemons.get_daemon('mds', mds).restart(*args) def wait_for_stable(self, rank = None, gid = None): -- 2.39.5