mgr/cephadm: don't allow upgrade start with less than 2 mgrs

Fixes: https://tracker.ceph.com/issues/50443
Signed-off-by: Adam King <adking@redhat.com>
(cherry picked from commit 06d767ee6d36dc5f74044dc3a030c5c46cb09cbe)

diff --git a/src/pybind/mgr/cephadm/tests/test_upgrade.py b/src/pybind/mgr/cephadm/tests/test_upgrade.py
index 9af6eb3399194265d370336a8fbb6f422145240f..0751192df0d04fb65287663bc4c548e90123a0f3 100644 (file)
--- a/src/pybind/mgr/cephadm/tests/test_upgrade.py
+++ b/src/pybind/mgr/cephadm/tests/test_upgrade.py
@@ -7,23 +7,29 @@ from ceph.deployment.service_spec import PlacementSpec, ServiceSpec
 from cephadm import CephadmOrchestrator
 from cephadm.upgrade import CephadmUpgrade
 from cephadm.serve import CephadmServe
+from orchestrator import OrchestratorError
 from .fixtures import _run_cephadm, wait, with_host, with_service
 
 
 @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}'))
 def test_upgrade_start(cephadm_module: CephadmOrchestrator):
     with with_host(cephadm_module, 'test'):
-        assert wait(cephadm_module, cephadm_module.upgrade_start(
-            'image_id', None)) == 'Initiating upgrade to docker.io/image_id'
+        with with_host(cephadm_module, 'test2'):
+            with with_service(cephadm_module, ServiceSpec('mgr', placement=PlacementSpec(count=2))):
+                assert wait(cephadm_module, cephadm_module.upgrade_start(
+                    'image_id', None)) == 'Initiating upgrade to docker.io/image_id'
 
-        assert wait(cephadm_module, cephadm_module.upgrade_status()).target_image == 'docker.io/image_id'
+                assert wait(cephadm_module, cephadm_module.upgrade_status()
+                            ).target_image == 'docker.io/image_id'
 
-        assert wait(cephadm_module, cephadm_module.upgrade_pause()) == 'Paused upgrade to docker.io/image_id'
+                assert wait(cephadm_module, cephadm_module.upgrade_pause()
+                            ) == 'Paused upgrade to docker.io/image_id'
 
-        assert wait(cephadm_module, cephadm_module.upgrade_resume()
-                    ) == 'Resumed upgrade to docker.io/image_id'
+                assert wait(cephadm_module, cephadm_module.upgrade_resume()
+                            ) == 'Resumed upgrade to docker.io/image_id'
 
-        assert wait(cephadm_module, cephadm_module.upgrade_stop()) == 'Stopped upgrade to docker.io/image_id'
+                assert wait(cephadm_module, cephadm_module.upgrade_stop()
+                            ) == 'Stopped upgrade to docker.io/image_id'
 
 
 @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}'))
@@ -114,3 +120,11 @@ def test_upgrade_state_null(cephadm_module: CephadmOrchestrator):
     cephadm_module.set_store('upgrade_state', 'null')
     CephadmUpgrade(cephadm_module)
     assert CephadmUpgrade(cephadm_module).upgrade_state is None
+
+
+@mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}'))
+def test_not_enough_mgrs(cephadm_module: CephadmOrchestrator):
+    with with_host(cephadm_module, 'host1'):
+        with with_service(cephadm_module, ServiceSpec('mgr', placement=PlacementSpec(count=1)), CephadmOrchestrator.apply_mgr, ''):
+            with pytest.raises(OrchestratorError):
+                wait(cephadm_module, cephadm_module.upgrade_start('image_id', None))

diff --git a/src/pybind/mgr/cephadm/upgrade.py b/src/pybind/mgr/cephadm/upgrade.py
index 732e1fae9c49474ab9e33b7f347fbee484641b6a..6ac76b4c8523b818cd682fd5c9490c8edf527189 100644 (file)
--- a/src/pybind/mgr/cephadm/upgrade.py
+++ b/src/pybind/mgr/cephadm/upgrade.py
@@ -8,7 +8,7 @@ import orchestrator
 from cephadm.serve import CephadmServe
 from cephadm.services.cephadmservice import CephadmDaemonDeploySpec
 from cephadm.utils import ceph_release_to_major, name_to_config_section, CEPH_UPGRADE_ORDER, MONITORING_STACK_TYPES
-from orchestrator import OrchestratorError, DaemonDescription, daemon_type_to_service
+from orchestrator import OrchestratorError, DaemonDescription, DaemonDescriptionStatus, daemon_type_to_service
 
 if TYPE_CHECKING:
     from .module import CephadmOrchestrator
@@ -199,6 +199,13 @@ class CephadmUpgrade:
                 self._save_upgrade_state()
                 return 'Resumed upgrade to %s' % self.target_image
             return 'Upgrade to %s in progress' % self.target_image
+
+        running_mgr_count = len([daemon for daemon in self.mgr.cache.get_daemons_by_type(
+            'mgr') if daemon.status == DaemonDescriptionStatus.running])
+
+        if running_mgr_count < 2:
+            raise OrchestratorError('Need at least 2 running mgr daemons for upgrade')
+
         self.mgr.log.info('Upgrade: Started with target %s' % target_name)
         self.upgrade_state = UpgradeState(
             target_name=target_name,