mgr/test_orchestrator: refactor listing services/daemons

- Refactor list_daemons: return ceph-xxx processes on host.
- Refactor describe_service: return services by grouping ceph-xxx
  processes on host.
- Add dummy data.

Signed-off-by: Kiefer Chang <kiefer.chang@suse.com>

diff --git a/src/pybind/mgr/test_orchestrator/dummy_data.json b/src/pybind/mgr/test_orchestrator/dummy_data.json
index 477655a365d65ee7092826b7039c1e1a4fd62b9d..3068979ceb1268d69e7ec1882b9770455c43d1b5 100644 (file)
--- a/src/pybind/mgr/test_orchestrator/dummy_data.json
+++ b/src/pybind/mgr/test_orchestrator/dummy_data.json
@@ -145,11 +145,124 @@
       ]
     }
   ],
+  "services": [
+    {
+      "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
+      "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
+      "service_name": "crash",
+      "size": 1,
+      "running": 0,
+      "last_refresh": "2020-02-26T07:23:56.501157"
+    },
+    {
+      "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
+      "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
+      "service_name": "mgr",
+      "size": 1,
+      "running": 1,
+      "last_refresh": "2020-02-26T07:23:56.501087"
+    },
+    {
+      "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
+      "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
+      "service_name": "mon",
+      "size": 1,
+      "running": 1,
+      "last_refresh": "2020-02-26T07:23:56.501013"
+    },
+    {
+      "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
+      "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
+      "service_name": "osd",
+      "size": 4,
+      "running": 4,
+      "last_refresh": "2020-02-26T07:23:56.500960"
+    }
+  ],
   "daemons": [
     {
       "hostname": "host0",
+      "container_id": "a8ad2d6ceb5d5b8a308207ea58963b9295d52d3c282a054534b1ed52e5e77d31",
+      "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
+      "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
+      "daemon_id": "host0",
+      "daemon_type": "crash",
+      "version": "15.1.0-1240-ge5841ce",
+      "status": -1,
+      "status_desc": "unknown",
+      "last_refresh": "2020-02-26T07:23:56.501157"
+    },
+    {
+      "hostname": "host0",
+      "container_id": "10f182e93c9c9110b7223e7b8e0337445fded77a317c649fc54b188f22c6e4cb",
+      "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
+      "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
+      "daemon_id": "host0.kjapjg",
+      "daemon_type": "mgr",
+      "version": "15.1.0-1240-ge5841ce",
+      "status": 1,
+      "status_desc": "running",
+      "last_refresh": "2020-02-26T07:23:56.501087"
+    },
+    {
+      "hostname": "host0",
+      "container_id": "27f178927ca4dbbd8d73380662f15828f780604f97baae4cac4d3f984e2c32af",
+      "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
+      "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
+      "daemon_id": "host0",
+      "daemon_type": "mon",
+      "version": "15.1.0-1240-ge5841ce",
+      "status": 1,
+      "status_desc": "running",
+      "last_refresh": "2020-02-26T07:23:56.501013"
+    },
+    {
+      "hostname": "host0",
+      "container_id": "71803dddfb9f736c26cc2127e684c66486034292ea94bc560d00132c29fe8b16",
+      "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
+      "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
+      "daemon_id": "0",
+      "daemon_type": "osd",
+      "version": "15.1.0-1240-ge5841ce",
+      "status": 1,
+      "status_desc": "running",
+      "last_refresh": "2020-02-26T07:23:56.501050"
+    },
+    {
+      "hostname": "host0",
+      "container_id": "3ca816209df390e8de3cdff78d95f69efd664b4017036b4d7c47563731e1fe37",
+      "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
+      "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
+      "daemon_id": "1",
+      "daemon_type": "osd",
+      "version": "15.1.0-1240-ge5841ce",
+      "status": 1,
+      "status_desc": "running",
+      "last_refresh": "2020-02-26T07:23:56.500960"
+    },
+    {
+      "hostname": "host0",
+      "container_id": "fcbcc88f023f3734f9015fcdae09277ed4c5e6b0ed1590275e4538d897588acb",
+      "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
+      "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
+      "daemon_id": "2",
+      "daemon_type": "osd",
+      "version": "15.1.0-1240-ge5841ce",
+      "status": 1,
+      "status_desc": "running",
+      "last_refresh": "2020-02-26T07:23:56.501123"
+    },
+    {
+      "hostname": "host0",
+      "container_id": "55f3894fffa52854e22354c7629cc4c58db46903ac0d977988b81ed5ba4ad759",
+      "container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
+      "container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
+      "daemon_id": "3",
       "daemon_type": "osd",
-      "daemon_id": "1"
+      "version": "15.1.0-1240-ge5841ce",
+      "status": 1,
+      "status_desc": "running",
+      "last_refresh": "2020-02-26T07:23:56.501193"
     }
   ]
 }

diff --git a/src/pybind/mgr/test_orchestrator/module.py b/src/pybind/mgr/test_orchestrator/module.py
index 70b556c61d92cc5cf6907855dffccd65e44c50fc..ee375824098bcfd9e26009870fa41d58972bff75 100644 (file)
--- a/src/pybind/mgr/test_orchestrator/module.py
+++ b/src/pybind/mgr/test_orchestrator/module.py
@@ -4,6 +4,7 @@ import re
 import os
 import threading
 import functools
+import itertools
 from subprocess import check_output, CalledProcessError
 try:
     from typing import Callable, List, Tuple
@@ -117,8 +118,10 @@ class TestOrchestrator(MgrModule, orchestrator.Orchestrator):
     def _init_data(self, data=None):
         self._inventory = [orchestrator.InventoryHost.from_json(inventory_host)
                            for inventory_host in data.get('inventory', [])]
-        self._daemons = [orchestrator.DaemonDescription.from_json(service)
-                          for service in data.get('daemons', [])]
+        self._services = [orchestrator.ServiceDescription.from_json(service)
+                           for service in data.get('services', [])]
+        self._daemons = [orchestrator.DaemonDescription.from_json(daemon)
+                          for daemon in data.get('daemons', [])]
 
     @deferred_read
     def get_inventory(self, host_filter=None, refresh=False):
@@ -153,6 +156,60 @@ class TestOrchestrator(MgrModule, orchestrator.Orchestrator):
         self.log.error('c-v failed: ' + str(c_v_out))
         raise Exception('c-v failed')
 
+    def _get_ceph_daemons(self):
+        # type: () -> List[orchestrator.DaemonDescription]
+        """ Return ceph daemons on the running host."""
+        types = ("mds", "osd", "mon", "rgw", "mgr")
+        out = map(str, check_output(['ps', 'aux']).splitlines())
+        processes = [p for p in out if any(
+            [('ceph-' + t in p) for t in types])]
+
+        daemons = []
+        for p in processes:
+            daemon = orchestrator.DaemonDescription()
+            # parse daemon type
+            m = re.search('ceph-([^ ]+)', p)
+            if m:
+                _daemon_type = m.group(1)
+            else:
+                raise AssertionError('Fail to determine daemon type from {}'.format(p))
+
+            # parse daemon ID. Possible options: `-i <id>`, `--id=<id>`, `--id <id>`
+            patterns = ['-i\s(\w+)', '--id[\s=](\w+)']
+            daemon_id = None
+            for pattern in patterns:
+                m = re.search(pattern, p)
+                if m:
+                    daemon_id = m.group(1)
+                    break
+            else:
+                raise AssertionError('Fail to determine daemon ID from {}'.format(p))
+            daemon = orchestrator.DaemonDescription(
+                daemon_type=_daemon_type, daemon_id=daemon_id, hostname='localhost')
+            daemons.append(daemon)
+        return daemons
+
+    @deferred_read
+    def describe_service(self, service_type=None, service_name=None, refresh=False):
+        if self._services:
+            # Dummy data
+            services = self._services
+        else:
+            # Deduce services from daemons running on localhost
+            all_daemons = self._get_ceph_daemons()
+            services = []
+            for daemon_type, daemons in itertools.groupby(all_daemons, key=lambda d: d.daemon_type):
+                daemon_size = len(list(daemons))
+                services.append(orchestrator.ServiceDescription(
+                    service_name=daemon_type, size=daemon_size, running=daemon_size))
+        
+        def _filter_func(svc):
+            if service_type is not None and service_type != svc.service_name:
+                return False
+            return True
+
+        return list(filter(_filter_func, services))
+
     @deferred_read
     def list_daemons(self, daemon_type=None, daemon_id=None, host=None, refresh=False):
         """
@@ -160,29 +217,21 @@ class TestOrchestrator(MgrModule, orchestrator.Orchestrator):
         it returns the mgr we're running in.
         """
         if daemon_type:
-            daemon_types = ("mds", "osd", "mon", "rgw", "mgr", "iscsi")
+            daemon_types = ("mds", "osd", "mon", "rgw", "mgr", "iscsi", "crash")
             assert daemon_type in daemon_types, daemon_type + " unsupported"
 
-        if self._daemons:
-            if host:
-                return list(filter(lambda svc: svc.hostname == host, self._daemons))
-            return self._daemons
+        daemons = self._daemons if self._daemons else self._get_ceph_daemons()
 
-        out = map(str, check_output(['ps', 'aux']).splitlines())
-        types = (daemon_type, ) if daemon_type else ("mds", "osd", "mon", "rgw", "mgr")
-        assert isinstance(types, tuple)
-        processes = [p for p in out if any([('ceph-' + t in p) for t in types])]
+        def _filter_func(d):
+            if daemon_type is not None and daemon_type != d.daemon_type:
+                return False
+            if daemon_id is not None and daemon_id != d.daemon_id:
+                return False
+            if host is not None and host != d.hostname:
+                return False
+            return True
 
-        result = []
-        for p in processes:
-            sd = orchestrator.DaemonDescription()
-            sd.hostname = 'localhost'
-            res = re.search('ceph-[^ ]+', p)
-            assert res
-            sd.daemon_id = res.group()
-            result.append(sd)
-
-        return result
+        return list(filter(_filter_func, daemons))
 
     def create_osds(self, drive_groups):
         # type: (List[DriveGroupSpec]) -> TestCompletion