ceph-volume: systemd should retry several times to activate a device

Allows environment variables to tweak the retries and intervals,
defaulting to 30 tries at 5 second intervals.

Signed-off-by: Alfredo Deza <adeza@redhat.com>

diff --git a/src/ceph-volume/ceph_volume/systemd/main.py b/src/ceph-volume/ceph_volume/systemd/main.py
index 5cb4fbf6c7f0c929915f03de648243515f9eab9d..990d68b12f988e03912a9a27779c37758feeeafd 100644 (file)
--- a/src/ceph-volume/ceph_volume/systemd/main.py
+++ b/src/ceph-volume/ceph_volume/systemd/main.py
@@ -3,8 +3,9 @@ This file is used only by systemd units that are passing their instance suffix
 as arguments to this script so that it can parse the suffix into arguments that
 ``ceph-volume <sub command>`` can consume
 """
-
+import os
 import sys
+import time
 import logging
 from ceph_volume import log, process
 from ceph_volume.exceptions import SuffixParsingError
@@ -74,5 +75,17 @@ def main(args=None):
     logger.info('osd id: %s, osd uuid: %s, sub-command: %s', osd_id, osd_uuid, sub_command)
     command = ['ceph-volume', sub_command, 'activate', osd_id, osd_uuid]
 
-    # don't log any output to the terminal, just rely on stderr/stdout going to logging
-    process.run(command, terminal_logging=False)
+    tries = os.environ.get('CEPH_VOLUME_SYSTEMD_TRIES', 30)
+    interval = os.environ.get('CEPH_VOLUME_SYSTEMD_INTERVAL', 5)
+    while tries > 0:
+        try:
+            # don't log any output to the terminal, just rely on stderr/stdout
+            # going to logging
+            process.run(command, terminal_logging=False)
+            logger.info('successfully activated OSD %s %s', osd_id, osd_uuid)
+            break
+        except RuntimeError as error:
+            logger.warning(error)
+            logger.warning('failed activating OSD, retries left: %s', tries)
+            tries -= 1
+            time.sleep(interval)