mgr/cephadm: track last_applied by host for osd specs

For each host, note when we last applied each osdspec.  Log the start
time, not the end time.

Signed-off-by: Sage Weil <sage@newdream.net>
(cherry picked from commit 197a8ba22ff30ccf9498bbc14b7a3897e48e1220)

diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index 1629738444b834dda71583c70fa7cab251a34a19..189be8adc52dc370fe6168c3c73fe07eec5497b0 100644 (file)
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -213,6 +213,7 @@ class HostCache():
         self.facts = {}                # type: Dict[str, Dict[str, Any]]
         self.last_facts_update = {}    # type: Dict[str, datetime.datetime]
         self.osdspec_previews = {}     # type: Dict[str, List[Dict[str, Any]]]
+        self.osdspec_last_applied = {} # type: Dict[str, Dict[str, datetime.datetime]]
         self.networks = {}             # type: Dict[str, Dict[str, List[str]]]
         self.last_device_update = {}   # type: Dict[str, datetime.datetime]
         self.daemon_refresh_queue = []  # type: List[str]
@@ -247,6 +248,7 @@ class HostCache():
                 self.daemon_refresh_queue.append(host)
                 self.daemons[host] = {}
                 self.osdspec_previews[host] = []
+                self.osdspec_last_applied[host] = {}
                 self.devices[host] = []
                 self.networks[host] = {}
                 self.daemon_config_deps[host] = {}
@@ -257,6 +259,8 @@ class HostCache():
                     self.devices[host].append(inventory.Device.from_json(d))
                 self.networks[host] = j.get('networks', {})
                 self.osdspec_previews[host] = j.get('osdspec_previews', {})
+                for name, ts in j.get('osdspec_last_applied', {}).items():
+                    self.osdspec_last_applied[host][name] = str_to_datetime(ts)
 
                 for name, d in j.get('daemon_config_deps', {}).items():
                     self.daemon_config_deps[host][name] = {
@@ -307,6 +311,10 @@ class HostCache():
         # type: (str) -> None
         self.last_host_check[host] = datetime_now()
 
+    def update_osdspec_last_applied(self, host, service_name, ts):
+        # type: (str, str, datetime.datetime) -> None
+        self.osdspec_last_applied[host][service_name] = ts
+
     def prime_empty_host(self, host):
         # type: (str) -> None
         """
@@ -316,6 +324,7 @@ class HostCache():
         self.devices[host] = []
         self.networks[host] = {}
         self.osdspec_previews[host] = []
+        self.osdspec_last_applied[host] = {}
         self.daemon_config_deps[host] = {}
         self.daemon_refresh_queue.append(host)
         self.device_refresh_queue.append(host)
@@ -344,6 +353,7 @@ class HostCache():
             'daemons': {},
             'devices': [],
             'osdspec_previews': [],
+            'osdspec_last_applied': {},
             'daemon_config_deps': {},
         }
         if host in self.last_daemon_update:
@@ -366,6 +376,9 @@ class HostCache():
                 }
         if host in self.osdspec_previews and self.osdspec_previews[host]:
             j['osdspec_previews'] = self.osdspec_previews[host]
+        if host in self.osdspec_last_applied:
+            for name, ts in self.osdspec_last_applied[host].items():
+                j['osdspec_last_applied'][name] = datetime_to_str(ts)
 
         if host in self.last_host_check:
             j['last_host_check'] = datetime_to_str(self.last_host_check[host])
@@ -389,6 +402,8 @@ class HostCache():
             del self.last_facts_update[host]
         if host in self.osdspec_previews:
             del self.osdspec_previews[host]
+        if host in self.osdspec_last_applied:
+            del self.osdspec_last_applied[host]
         if host in self.loading_osdspec_preview:
             self.loading_osdspec_preview.remove(host)
         if host in self.networks:

diff --git a/src/pybind/mgr/cephadm/services/osd.py b/src/pybind/mgr/cephadm/services/osd.py
index 8febf352b615bb4c67c1a421c21c14dd1d978ede..c4c9ecd12bc88195ca466a1976ce7dba48377d2d 100644 (file)
--- a/src/pybind/mgr/cephadm/services/osd.py
+++ b/src/pybind/mgr/cephadm/services/osd.py
@@ -41,17 +41,28 @@ class OSDService(CephService):
                 logger.debug("No data_devices, skipping DriveGroup: {}".format(
                     drive_group.service_id))
                 return None
-            logger.info('Applying drive group %s on host %s...' % (drive_group.service_id, host))
+
+            logger.info('Applying drive group %s on host %s...' % (
+                drive_group.service_id, host
+            ))
+            start_ts = datetime_now()
             env_vars: List[str] = [f"CEPH_VOLUME_OSDSPEC_AFFINITY={drive_group.service_id}"]
             ret_msg = self.create_single_host(
-                host, cmd, replace_osd_ids=osd_id_claims.get(host, []), env_vars=env_vars
+                drive_group, host, cmd,
+                replace_osd_ids=osd_id_claims.get(host, []), env_vars=env_vars
+            )
+            self.mgr.cache.update_osdspec_last_applied(
+                host, drive_group.service_name(), start_ts
             )
+            self.mgr.cache.save_host(host)
             return ret_msg
 
         ret = create_from_spec_one(self.prepare_drivegroup(drive_group))
         return ", ".join(filter(None, ret))
 
-    def create_single_host(self, host: str, cmd: str, replace_osd_ids: List[str],
+    def create_single_host(self,
+                           drive_group: DriveGroupSpec,
+                           host: str, cmd: str, replace_osd_ids: List[str],
                            env_vars: Optional[List[str]] = None) -> str:
         out, err, code = self._run_ceph_volume_command(host, cmd, env_vars=env_vars)