From 5e5ca94eb71b7ca28bbd80bcd120fc6826e05999 Mon Sep 17 00:00:00 2001 From: Adam King Date: Thu, 22 Apr 2021 17:39:32 -0400 Subject: [PATCH] mgr/cephadm: skip ok-to-stop for mons in upgrade if < 3 mons Fixes: https://tracker.ceph.com/issues/50443 Signed-off-by: Adam King (cherry picked from commit 4c7bfdbdd133820a1c0e8d129980ed810a77168d) --- src/pybind/mgr/cephadm/tests/test_upgrade.py | 14 ++++++++++++++ src/pybind/mgr/cephadm/upgrade.py | 19 ++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/cephadm/tests/test_upgrade.py b/src/pybind/mgr/cephadm/tests/test_upgrade.py index 0751192df0d04..c4d6c27a01e92 100644 --- a/src/pybind/mgr/cephadm/tests/test_upgrade.py +++ b/src/pybind/mgr/cephadm/tests/test_upgrade.py @@ -128,3 +128,17 @@ def test_not_enough_mgrs(cephadm_module: CephadmOrchestrator): with with_service(cephadm_module, ServiceSpec('mgr', placement=PlacementSpec(count=1)), CephadmOrchestrator.apply_mgr, ''): with pytest.raises(OrchestratorError): wait(cephadm_module, cephadm_module.upgrade_start('image_id', None)) + + +@mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) +@mock.patch("cephadm.CephadmOrchestrator.check_mon_command") +def test_enough_mons_for_ok_to_stop(check_mon_command, cephadm_module: CephadmOrchestrator): + # only 2 monitors, not enough for ok-to-stop to ever pass + check_mon_command.return_value = ( + 0, '{"monmap": {"mons": [{"name": "mon.1"}, {"name": "mon.2"}]}}', '') + assert not cephadm_module.upgrade._enough_mons_for_ok_to_stop() + + # 3 monitors, ok-to-stop should work fine + check_mon_command.return_value = ( + 0, '{"monmap": {"mons": [{"name": "mon.1"}, {"name": "mon.2"}, {"name": "mon.3"}]}}', '') + assert cephadm_module.upgrade._enough_mons_for_ok_to_stop() diff --git a/src/pybind/mgr/cephadm/upgrade.py b/src/pybind/mgr/cephadm/upgrade.py index 6ac76b4c8523b..830b1cb1a57b9 100644 --- a/src/pybind/mgr/cephadm/upgrade.py +++ b/src/pybind/mgr/cephadm/upgrade.py @@ -420,6 +420,19 @@ class CephadmUpgrade: return continue_upgrade + def _enough_mons_for_ok_to_stop(self) -> bool: + # type () -> bool + ret, out, err = self.mgr.check_mon_command({ + 'prefix': 'quorum_status', + }) + try: + j = json.loads(out) + except Exception: + raise OrchestratorError('failed to parse quorum status') + + mons = [m['name'] for m in j['monmap']['mons']] + return len(mons) > 2 + def _do_upgrade(self): # type: () -> None if not self.upgrade_state: @@ -567,12 +580,16 @@ class CephadmUpgrade: to_upgrade.append(d_entry) continue - if d.daemon_type in ['mon', 'osd', 'mds']: + if d.daemon_type in ['osd', 'mds']: # NOTE: known_ok_to_stop is an output argument for # _wait_for_ok_to_stop if not self._wait_for_ok_to_stop(d, known_ok_to_stop): return + if d.daemon_type == 'mon' and self._enough_mons_for_ok_to_stop(): + if not self._wait_for_ok_to_stop(d, known_ok_to_stop): + return + to_upgrade.append(d_entry) # if we don't have a list of others to consider, stop now -- 2.39.5