From: Adam King Date: Fri, 5 Mar 2021 22:01:12 +0000 (-0500) Subject: mgr/cephadm: add info to 'ceph orch upgrade status' in cephadm X-Git-Tag: v16.2.0~73^2~47 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d4001860d997ef30b92d27bcf8e54474c8cf7ad2;p=ceph.git mgr/cephadm: add info to 'ceph orch upgrade status' in cephadm properly fills in 'services_complete' field, adds info messages to 'message' field such as what daemon type is being upgraded or if we're pulling an image and adds 'progress' field that shows how many daemons have been upgraded so far. Fixes: https://tracker.ceph.com/issues/49235 Signed-off-by: Adam King (cherry picked from commit 1527b3a527c45deb689f3d7b5aff02e517357584) --- diff --git a/src/pybind/mgr/cephadm/upgrade.py b/src/pybind/mgr/cephadm/upgrade.py index 24b36fa8fc39..fddabac28672 100644 --- a/src/pybind/mgr/cephadm/upgrade.py +++ b/src/pybind/mgr/cephadm/upgrade.py @@ -2,7 +2,7 @@ import json import logging import time import uuid -from typing import TYPE_CHECKING, Optional, Dict, List +from typing import TYPE_CHECKING, Optional, Dict, List, Tuple import orchestrator from cephadm.serve import CephadmServe @@ -92,12 +92,38 @@ class CephadmUpgrade: if self.upgrade_state: r.target_image = self.target_image r.in_progress = True + r.progress, r.services_complete = self._get_upgrade_info() + # accessing self.upgrade_info_str will throw an exception if it + # has not been set in _do_upgrade yet + try: + r.message = self.upgrade_info_str + except AttributeError: + pass if self.upgrade_state.error: r.message = 'Error: ' + self.upgrade_state.error elif self.upgrade_state.paused: r.message = 'Upgrade paused' return r + def _get_upgrade_info(self) -> Tuple[str, List[str]]: + if not self.upgrade_state or not self.upgrade_state.target_digests: + return '', [] + + daemons = [d for d in self.mgr.cache.get_daemons() if d.daemon_type in CEPH_UPGRADE_ORDER] + + if any(not d.container_image_digests for d in daemons if d.daemon_type == 'mgr'): + return '', [] + + completed_daemons = [(d.daemon_type, any(d in self.upgrade_state.target_digests for d in ( + d.container_image_digests or []))) for d in daemons if d.daemon_type] + + done = len([True for completion in completed_daemons if completion[1]]) + + completed_types = list(set([completion[0] for completion in completed_daemons if all( + c[1] for c in completed_daemons if c[0] == completion[0])])) + + return '%s/%s ceph daemons upgraded' % (done, len(daemons)), completed_types + def _check_target_version(self, version: str) -> Optional[str]: try: (major, minor, patch) = version.split('.') @@ -370,6 +396,7 @@ class CephadmUpgrade: if not target_id or not target_version or not target_digests: # need to learn the container hash logger.info('Upgrade: First pull of %s' % target_image) + self.upgrade_info_str = 'Doing first pull of %s image' % (target_image) try: target_id, target_version, target_digests = CephadmServe(self.mgr)._get_container_image_info( target_image) @@ -418,7 +445,7 @@ class CephadmUpgrade: image_settings = self.get_distinct_container_image_settings() - daemons = self.mgr.cache.get_daemons() + daemons = [d for d in self.mgr.cache.get_daemons() if d.daemon_type in CEPH_UPGRADE_ORDER] done = 0 for daemon_type in CEPH_UPGRADE_ORDER: logger.info('Upgrade: Checking %s daemons' % daemon_type) @@ -456,6 +483,9 @@ class CephadmUpgrade: ): return + if need_upgrade: + self.upgrade_info_str = 'Currently upgrading %s daemons' % (daemon_type) + to_upgrade = [] known_ok_to_stop: List[str] = [] for d in need_upgrade: @@ -502,6 +532,8 @@ class CephadmUpgrade: if code or not any(d in target_digests for d in json.loads(''.join(out)).get('repo_digests', [])): logger.info('Upgrade: Pulling %s on %s' % (target_image, d.hostname)) + self.upgrade_info_str = 'Pulling %s image on host %s' % ( + target_image, d.hostname) out, errs, code = CephadmServe(self.mgr)._run_cephadm( d.hostname, '', 'pull', [], image=target_image, no_fsid=True, error_ok=True) @@ -519,10 +551,14 @@ class CephadmUpgrade: if not any(d in target_digests for d in r.get('repo_digests', [])): logger.info('Upgrade: image %s pull on %s got new digests %s (not %s), restarting' % ( target_image, d.hostname, r['repo_digests'], target_digests)) + self.upgrade_info_str = 'Image %s pull on %s got new digests %s (not %s), restarting' % ( + target_image, d.hostname, r['repo_digests'], target_digests) self.upgrade_state.target_digests = r['repo_digests'] self._save_upgrade_state() return + self.upgrade_info_str = 'Currently upgrading %s daemons' % (daemon_type) + if len(to_upgrade) > 1: logger.info('Upgrade: Updating %s.%s (%d/%d)' % (d.daemon_type, d.daemon_id, num, len(to_upgrade))) diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py index 29b8603818ea..11c70b0f8354 100644 --- a/src/pybind/mgr/orchestrator/_interface.py +++ b/src/pybind/mgr/orchestrator/_interface.py @@ -734,6 +734,7 @@ class UpgradeStatusSpec(object): self.in_progress = False # Is an upgrade underway? self.target_image: Optional[str] = None self.services_complete: List[str] = [] # Which daemon types are fully updated? + self.progress: Optional[str] = None # How many of the daemons have we upgraded self.message = "" # Freeform description diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index d314dc27ae11..bcffdc833e6c 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -1298,6 +1298,7 @@ Usage: 'target_image': status.target_image, 'in_progress': status.in_progress, 'services_complete': status.services_complete, + 'progress': status.progress, 'message': status.message, } out = json.dumps(r, indent=4)