From 229c08106f721840c5ea15a334339e0e33b2c143 Mon Sep 17 00:00:00 2001 From: Kiefer Chang Date: Thu, 27 Feb 2020 15:25:05 +0800 Subject: [PATCH] mgr/test_orchestrator: refactor listing services/daemons - Refactor list_daemons: return ceph-xxx processes on host. - Refactor describe_service: return services by grouping ceph-xxx processes on host. - Add dummy data. Signed-off-by: Kiefer Chang --- .../mgr/test_orchestrator/dummy_data.json | 115 +++++++++++++++++- src/pybind/mgr/test_orchestrator/module.py | 91 ++++++++++---- 2 files changed, 184 insertions(+), 22 deletions(-) diff --git a/src/pybind/mgr/test_orchestrator/dummy_data.json b/src/pybind/mgr/test_orchestrator/dummy_data.json index 477655a365d..3068979ceb1 100644 --- a/src/pybind/mgr/test_orchestrator/dummy_data.json +++ b/src/pybind/mgr/test_orchestrator/dummy_data.json @@ -145,11 +145,124 @@ ] } ], + "services": [ + { + "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944", + "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel", + "service_name": "crash", + "size": 1, + "running": 0, + "last_refresh": "2020-02-26T07:23:56.501157" + }, + { + "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944", + "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel", + "service_name": "mgr", + "size": 1, + "running": 1, + "last_refresh": "2020-02-26T07:23:56.501087" + }, + { + "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944", + "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel", + "service_name": "mon", + "size": 1, + "running": 1, + "last_refresh": "2020-02-26T07:23:56.501013" + }, + { + "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944", + "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel", + "service_name": "osd", + "size": 4, + "running": 4, + "last_refresh": "2020-02-26T07:23:56.500960" + } + ], "daemons": [ { "hostname": "host0", + "container_id": "a8ad2d6ceb5d5b8a308207ea58963b9295d52d3c282a054534b1ed52e5e77d31", + "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944", + "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel", + "daemon_id": "host0", + "daemon_type": "crash", + "version": "15.1.0-1240-ge5841ce", + "status": -1, + "status_desc": "unknown", + "last_refresh": "2020-02-26T07:23:56.501157" + }, + { + "hostname": "host0", + "container_id": "10f182e93c9c9110b7223e7b8e0337445fded77a317c649fc54b188f22c6e4cb", + "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944", + "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel", + "daemon_id": "host0.kjapjg", + "daemon_type": "mgr", + "version": "15.1.0-1240-ge5841ce", + "status": 1, + "status_desc": "running", + "last_refresh": "2020-02-26T07:23:56.501087" + }, + { + "hostname": "host0", + "container_id": "27f178927ca4dbbd8d73380662f15828f780604f97baae4cac4d3f984e2c32af", + "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944", + "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel", + "daemon_id": "host0", + "daemon_type": "mon", + "version": "15.1.0-1240-ge5841ce", + "status": 1, + "status_desc": "running", + "last_refresh": "2020-02-26T07:23:56.501013" + }, + { + "hostname": "host0", + "container_id": "71803dddfb9f736c26cc2127e684c66486034292ea94bc560d00132c29fe8b16", + "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944", + "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel", + "daemon_id": "0", + "daemon_type": "osd", + "version": "15.1.0-1240-ge5841ce", + "status": 1, + "status_desc": "running", + "last_refresh": "2020-02-26T07:23:56.501050" + }, + { + "hostname": "host0", + "container_id": "3ca816209df390e8de3cdff78d95f69efd664b4017036b4d7c47563731e1fe37", + "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944", + "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel", + "daemon_id": "1", + "daemon_type": "osd", + "version": "15.1.0-1240-ge5841ce", + "status": 1, + "status_desc": "running", + "last_refresh": "2020-02-26T07:23:56.500960" + }, + { + "hostname": "host0", + "container_id": "fcbcc88f023f3734f9015fcdae09277ed4c5e6b0ed1590275e4538d897588acb", + "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944", + "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel", + "daemon_id": "2", + "daemon_type": "osd", + "version": "15.1.0-1240-ge5841ce", + "status": 1, + "status_desc": "running", + "last_refresh": "2020-02-26T07:23:56.501123" + }, + { + "hostname": "host0", + "container_id": "55f3894fffa52854e22354c7629cc4c58db46903ac0d977988b81ed5ba4ad759", + "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944", + "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel", + "daemon_id": "3", "daemon_type": "osd", - "daemon_id": "1" + "version": "15.1.0-1240-ge5841ce", + "status": 1, + "status_desc": "running", + "last_refresh": "2020-02-26T07:23:56.501193" } ] } diff --git a/src/pybind/mgr/test_orchestrator/module.py b/src/pybind/mgr/test_orchestrator/module.py index 70b556c61d9..ee375824098 100644 --- a/src/pybind/mgr/test_orchestrator/module.py +++ b/src/pybind/mgr/test_orchestrator/module.py @@ -4,6 +4,7 @@ import re import os import threading import functools +import itertools from subprocess import check_output, CalledProcessError try: from typing import Callable, List, Tuple @@ -117,8 +118,10 @@ class TestOrchestrator(MgrModule, orchestrator.Orchestrator): def _init_data(self, data=None): self._inventory = [orchestrator.InventoryHost.from_json(inventory_host) for inventory_host in data.get('inventory', [])] - self._daemons = [orchestrator.DaemonDescription.from_json(service) - for service in data.get('daemons', [])] + self._services = [orchestrator.ServiceDescription.from_json(service) + for service in data.get('services', [])] + self._daemons = [orchestrator.DaemonDescription.from_json(daemon) + for daemon in data.get('daemons', [])] @deferred_read def get_inventory(self, host_filter=None, refresh=False): @@ -153,6 +156,60 @@ class TestOrchestrator(MgrModule, orchestrator.Orchestrator): self.log.error('c-v failed: ' + str(c_v_out)) raise Exception('c-v failed') + def _get_ceph_daemons(self): + # type: () -> List[orchestrator.DaemonDescription] + """ Return ceph daemons on the running host.""" + types = ("mds", "osd", "mon", "rgw", "mgr") + out = map(str, check_output(['ps', 'aux']).splitlines()) + processes = [p for p in out if any( + [('ceph-' + t in p) for t in types])] + + daemons = [] + for p in processes: + daemon = orchestrator.DaemonDescription() + # parse daemon type + m = re.search('ceph-([^ ]+)', p) + if m: + _daemon_type = m.group(1) + else: + raise AssertionError('Fail to determine daemon type from {}'.format(p)) + + # parse daemon ID. Possible options: `-i `, `--id=`, `--id ` + patterns = ['-i\s(\w+)', '--id[\s=](\w+)'] + daemon_id = None + for pattern in patterns: + m = re.search(pattern, p) + if m: + daemon_id = m.group(1) + break + else: + raise AssertionError('Fail to determine daemon ID from {}'.format(p)) + daemon = orchestrator.DaemonDescription( + daemon_type=_daemon_type, daemon_id=daemon_id, hostname='localhost') + daemons.append(daemon) + return daemons + + @deferred_read + def describe_service(self, service_type=None, service_name=None, refresh=False): + if self._services: + # Dummy data + services = self._services + else: + # Deduce services from daemons running on localhost + all_daemons = self._get_ceph_daemons() + services = [] + for daemon_type, daemons in itertools.groupby(all_daemons, key=lambda d: d.daemon_type): + daemon_size = len(list(daemons)) + services.append(orchestrator.ServiceDescription( + service_name=daemon_type, size=daemon_size, running=daemon_size)) + + def _filter_func(svc): + if service_type is not None and service_type != svc.service_name: + return False + return True + + return list(filter(_filter_func, services)) + @deferred_read def list_daemons(self, daemon_type=None, daemon_id=None, host=None, refresh=False): """ @@ -160,29 +217,21 @@ class TestOrchestrator(MgrModule, orchestrator.Orchestrator): it returns the mgr we're running in. """ if daemon_type: - daemon_types = ("mds", "osd", "mon", "rgw", "mgr", "iscsi") + daemon_types = ("mds", "osd", "mon", "rgw", "mgr", "iscsi", "crash") assert daemon_type in daemon_types, daemon_type + " unsupported" - if self._daemons: - if host: - return list(filter(lambda svc: svc.hostname == host, self._daemons)) - return self._daemons + daemons = self._daemons if self._daemons else self._get_ceph_daemons() - out = map(str, check_output(['ps', 'aux']).splitlines()) - types = (daemon_type, ) if daemon_type else ("mds", "osd", "mon", "rgw", "mgr") - assert isinstance(types, tuple) - processes = [p for p in out if any([('ceph-' + t in p) for t in types])] + def _filter_func(d): + if daemon_type is not None and daemon_type != d.daemon_type: + return False + if daemon_id is not None and daemon_id != d.daemon_id: + return False + if host is not None and host != d.hostname: + return False + return True - result = [] - for p in processes: - sd = orchestrator.DaemonDescription() - sd.hostname = 'localhost' - res = re.search('ceph-[^ ]+', p) - assert res - sd.daemon_id = res.group() - result.append(sd) - - return result + return list(filter(_filter_func, daemons)) def create_osds(self, drive_groups): # type: (List[DriveGroupSpec]) -> TestCompletion -- 2.39.5