From d7717256644e45ba670165c6af0941461fb884f4 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Wed, 27 Oct 2021 20:26:55 -0400 Subject: [PATCH] pybind/mgr/cephadm: upgrade MDS if no MDS is "up" The upgrade process can get stuck if an MDS crashes. This should be rare when straddling v16.2.5 where the compatset of the file system inherits the FSMap "default". The MDS from pre-v16.2.5 do not yet share a compatset with the mons so the mons will do no promotions, causing upgrade task to get stuck. Fixes: https://tracker.ceph.com/issues/53074 Signed-off-by: Patrick Donnelly --- src/pybind/mgr/cephadm/upgrade.py | 32 +++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/pybind/mgr/cephadm/upgrade.py b/src/pybind/mgr/cephadm/upgrade.py index 0984a5f32c576..1cebe10651515 100644 --- a/src/pybind/mgr/cephadm/upgrade.py +++ b/src/pybind/mgr/cephadm/upgrade.py @@ -441,23 +441,31 @@ class CephadmUpgrade: continue_upgrade = False continue - if not (mdsmap['in'] == [0] and len(mdsmap['up']) == 1): + if not (mdsmap['in'] == [0] and len(mdsmap['up']) <= 1): self.mgr.log.info('Upgrade: Waiting for fs %s to scale down to reach 1 MDS' % (fs_name)) time.sleep(10) continue_upgrade = False continue - mdss = list(mdsmap['info'].values()) - assert len(mdss) == 1 - lone_mds = mdss[0] - if lone_mds['state'] != 'up:active': - self.mgr.log.info('Upgrade: Waiting for mds.%s to be up:active (currently %s)' % ( - lone_mds['name'], - lone_mds['state'], - )) - time.sleep(10) - continue_upgrade = False - continue + if len(mdsmap['up']) == 0: + self.mgr.log.warning("Upgrade: No mds is up; continuing upgrade procedure to poke things in the right direction") + # This can happen because the current version MDS have + # incompatible compatsets; the mons will not do any promotions. + # We must upgrade to continue. + elif len(mdsmap['up']) > 0: + mdss = list(mdsmap['info'].values()) + assert len(mdss) == 1 + lone_mds = mdss[0] + if lone_mds['state'] != 'up:active': + self.mgr.log.info('Upgrade: Waiting for mds.%s to be up:active (currently %s)' % ( + lone_mds['name'], + lone_mds['state'], + )) + time.sleep(10) + continue_upgrade = False + continue + else: + assert False return continue_upgrade -- 2.39.5