From ed85f8c27e3ca3bafe2da2a15dc4e93f60576252 Mon Sep 17 00:00:00 2001 From: Daniel-Pivonka Date: Wed, 13 Jan 2021 16:40:52 -0500 Subject: [PATCH] mgr/cephadm: ok-to-stop for mgr depends on #38854 Signed-off-by: Daniel-Pivonka (cherry picked from commit 96dcb1547c77e03c04accd660defa586825d94cc) --- .../mgr/cephadm/services/cephadmservice.py | 29 +++- src/pybind/mgr/cephadm/tests/test_upgrade.py | 146 +++++++++--------- 2 files changed, 99 insertions(+), 76 deletions(-) diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py index d9fee1270ab0b..da298b3e15637 100644 --- a/src/pybind/mgr/cephadm/services/cephadmservice.py +++ b/src/pybind/mgr/cephadm/services/cephadmservice.py @@ -1,3 +1,4 @@ +import errno import json import re import logging @@ -231,7 +232,7 @@ class CephadmService(metaclass=ABCMeta): logger.info(out) return HandleCommandResult(r.retval, out, r.stderr) - def _enough_daemons_to_stop(self, daemon_type: str, daemon_ids: List[str], service: str, low_limit: int) -> Tuple[bool, str]: + def _enough_daemons_to_stop(self, daemon_type: str, daemon_ids: List[str], service: str, low_limit: int, alert: bool = False) -> Tuple[bool, str]: # Provides a warning about if it possible or not to stop daemons in a service names = [f'{daemon_type}.{d_id}' for d_id in daemon_ids] number_of_running_daemons = len( @@ -247,9 +248,14 @@ class CephadmService(metaclass=ABCMeta): daemon_count = "only" if number_of_running_daemons == 1 else number_of_running_daemons left_count = "no" if num_daemons_left == 0 else num_daemons_left - out = (f'WARNING: Stopping {len(daemon_ids)} out of {number_of_running_daemons} daemons in {service} service. ' - f'Service will not be operational with {left_count} {plural(num_daemons_left)} left. ' - f'At least {low_limit} {plural(low_limit)} must be running to guarantee service. ') + if alert: + out = (f'ALERT: Cannot stop {names} in {service} service. ' + f'Not enough remaining {service} daemons. ' + f'Please deploy at least {low_limit + 1} {service} daemons before stopping {names}. ') + else: + out = (f'WARNING: Stopping {len(daemon_ids)} out of {number_of_running_daemons} daemons in {service} service. ' + f'Service will not be operational with {left_count} {plural(num_daemons_left)} left. ' + f'At least {low_limit} {plural(low_limit)} must be running to guarantee service. ') return True, out def pre_remove(self, daemon: DaemonDescription) -> None: @@ -521,6 +527,21 @@ class MgrService(CephService): num = len(mgr_map.get('standbys')) return bool(num) + def ok_to_stop(self, daemon_ids: List[str], force: bool = False) -> HandleCommandResult: + # ok to stop if there is more than 1 mgr and not trying to stop the active mgr + + warn, warn_message = self._enough_daemons_to_stop(self.TYPE, daemon_ids, 'Mgr', 1, True) + if warn: + return HandleCommandResult(-errno.EBUSY, '', warn_message) + + mgr_daemons = self.mgr.cache.get_daemons_by_type(self.TYPE) + active = self.get_active_daemon(mgr_daemons).daemon_id + if active in daemon_ids: + warn_message = 'ALERT: Cannot stop active Mgr daemon, Please switch active Mgrs with \'ceph mgr fail %s\'' % active + return HandleCommandResult(-errno.EBUSY, '', warn_message) + + return HandleCommandResult(0, warn_message, '') + class MdsService(CephService): TYPE = 'mds' diff --git a/src/pybind/mgr/cephadm/tests/test_upgrade.py b/src/pybind/mgr/cephadm/tests/test_upgrade.py index 7d4f091e9c5fc..420fbdd652c18 100644 --- a/src/pybind/mgr/cephadm/tests/test_upgrade.py +++ b/src/pybind/mgr/cephadm/tests/test_upgrade.py @@ -3,7 +3,7 @@ from unittest import mock import pytest -from ceph.deployment.service_spec import ServiceSpec +from ceph.deployment.service_spec import PlacementSpec, ServiceSpec from cephadm import CephadmOrchestrator from cephadm.upgrade import CephadmUpgrade from cephadm.serve import CephadmServe @@ -33,78 +33,80 @@ def test_upgrade_start(cephadm_module: CephadmOrchestrator): True ]) def test_upgrade_run(use_repo_digest, cephadm_module: CephadmOrchestrator): - with with_host(cephadm_module, 'test'): - cephadm_module.set_container_image('global', 'from_image') - if use_repo_digest: - cephadm_module.use_repo_digest = True - with with_service(cephadm_module, ServiceSpec('mgr'), CephadmOrchestrator.apply_mgr, 'test'),\ - mock.patch("cephadm.module.CephadmOrchestrator.lookup_release_name", - return_value='foo'),\ - mock.patch("cephadm.module.CephadmOrchestrator.version", - new_callable=mock.PropertyMock) as version_mock,\ - mock.patch("cephadm.module.CephadmOrchestrator.get", - return_value={ - # capture fields in both mon and osd maps - "require_osd_release": "pacific", - "min_mon_release": 16, - }): - version_mock.return_value = 'ceph version 18.2.1 (somehash)' - assert wait(cephadm_module, cephadm_module.upgrade_start( - 'to_image', None)) == 'Initiating upgrade to to_image' - - assert wait(cephadm_module, cephadm_module.upgrade_status()).target_image == 'to_image' - - def _versions_mock(cmd): - return json.dumps({ - 'mgr': { - 'ceph version 1.2.3 (asdf) blah': 1 - } - }) - - cephadm_module._mon_command_mock_versions = _versions_mock - - with mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm(json.dumps({ - 'image_id': 'image_id', - 'repo_digests': ['to_image@repo_digest'], - 'ceph_version': 'ceph version 18.2.3 (hash)', - }))): - - cephadm_module.upgrade._do_upgrade() - - assert cephadm_module.upgrade_status is not None - - with mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm( - json.dumps([ - dict( - name=list(cephadm_module.cache.daemons['test'].keys())[0], - style='cephadm', - fsid='fsid', - container_id='container_id', - container_image_id='image_id', - container_image_digests=['to_image@repo_digest'], - version='version', - state='running', - ) - ]) - )): - CephadmServe(cephadm_module)._refresh_hosts_and_daemons() - - with mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm(json.dumps({ - 'image_id': 'image_id', - 'repo_digests': ['to_image@repo_digest'], - 'ceph_version': 'ceph version 18.2.3 (hash)', - }))): - cephadm_module.upgrade._do_upgrade() - - _, image, _ = cephadm_module.check_mon_command({ - 'prefix': 'config get', - 'who': 'global', - 'key': 'container_image', - }) + with with_host(cephadm_module, 'host1'): + with with_host(cephadm_module, 'host2'): + cephadm_module.set_container_image('global', 'from_image') if use_repo_digest: - assert image == 'to_image@repo_digest' - else: - assert image == 'to_image' + cephadm_module.use_repo_digest = True + with with_service(cephadm_module, ServiceSpec('mgr', placement=PlacementSpec(host_pattern='*', count=2)), CephadmOrchestrator.apply_mgr, ''),\ + mock.patch("cephadm.module.CephadmOrchestrator.lookup_release_name", + return_value='foo'),\ + mock.patch("cephadm.module.CephadmOrchestrator.version", + new_callable=mock.PropertyMock) as version_mock,\ + mock.patch("cephadm.module.CephadmOrchestrator.get", + return_value={ + # capture fields in both mon and osd maps + "require_osd_release": "pacific", + "min_mon_release": 16, + }): + version_mock.return_value = 'ceph version 18.2.1 (somehash)' + assert wait(cephadm_module, cephadm_module.upgrade_start( + 'to_image', None)) == 'Initiating upgrade to to_image' + + assert wait(cephadm_module, cephadm_module.upgrade_status() + ).target_image == 'to_image' + + def _versions_mock(cmd): + return json.dumps({ + 'mgr': { + 'ceph version 1.2.3 (asdf) blah': 1 + } + }) + + cephadm_module._mon_command_mock_versions = _versions_mock + + with mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm(json.dumps({ + 'image_id': 'image_id', + 'repo_digests': ['to_image@repo_digest'], + 'ceph_version': 'ceph version 18.2.3 (hash)', + }))): + + cephadm_module.upgrade._do_upgrade() + + assert cephadm_module.upgrade_status is not None + + with mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm( + json.dumps([ + dict( + name=list(cephadm_module.cache.daemons['host1'].keys())[0], + style='cephadm', + fsid='fsid', + container_id='container_id', + container_image_id='image_id', + container_image_digests=['to_image@repo_digest'], + version='version', + state='running', + ) + ]) + )): + CephadmServe(cephadm_module)._refresh_hosts_and_daemons() + + with mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm(json.dumps({ + 'image_id': 'image_id', + 'repo_digests': ['to_image@repo_digest'], + 'ceph_version': 'ceph version 18.2.3 (hash)', + }))): + cephadm_module.upgrade._do_upgrade() + + _, image, _ = cephadm_module.check_mon_command({ + 'prefix': 'config get', + 'who': 'global', + 'key': 'container_image', + }) + if use_repo_digest: + assert image == 'to_image@repo_digest' + else: + assert image == 'to_image' def test_upgrade_state_null(cephadm_module: CephadmOrchestrator): -- 2.39.5