From: Alfredo Deza Date: Thu, 27 Jul 2017 11:46:18 +0000 (-0400) Subject: ceph-volume: systemd should retry several times to activate a device X-Git-Tag: ses5-milestone10~3^2~5^2~12 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2f187ec7a96489c1104ad5a10c2901dcae0b2a3d;p=ceph.git ceph-volume: systemd should retry several times to activate a device Allows environment variables to tweak the retries and intervals, defaulting to 30 tries at 5 second intervals. Signed-off-by: Alfredo Deza --- diff --git a/src/ceph-volume/ceph_volume/systemd/main.py b/src/ceph-volume/ceph_volume/systemd/main.py index 5cb4fbf6c7f0..990d68b12f98 100644 --- a/src/ceph-volume/ceph_volume/systemd/main.py +++ b/src/ceph-volume/ceph_volume/systemd/main.py @@ -3,8 +3,9 @@ This file is used only by systemd units that are passing their instance suffix as arguments to this script so that it can parse the suffix into arguments that ``ceph-volume `` can consume """ - +import os import sys +import time import logging from ceph_volume import log, process from ceph_volume.exceptions import SuffixParsingError @@ -74,5 +75,17 @@ def main(args=None): logger.info('osd id: %s, osd uuid: %s, sub-command: %s', osd_id, osd_uuid, sub_command) command = ['ceph-volume', sub_command, 'activate', osd_id, osd_uuid] - # don't log any output to the terminal, just rely on stderr/stdout going to logging - process.run(command, terminal_logging=False) + tries = os.environ.get('CEPH_VOLUME_SYSTEMD_TRIES', 30) + interval = os.environ.get('CEPH_VOLUME_SYSTEMD_INTERVAL', 5) + while tries > 0: + try: + # don't log any output to the terminal, just rely on stderr/stdout + # going to logging + process.run(command, terminal_logging=False) + logger.info('successfully activated OSD %s %s', osd_id, osd_uuid) + break + except RuntimeError as error: + logger.warning(error) + logger.warning('failed activating OSD, retries left: %s', tries) + tries -= 1 + time.sleep(interval)