Add option to retry for exec task.

retry and sleep_for_retry options can be used with exec task without
needing to use lengthy shell based while loops for similar
task. Also adds timeout option for commands that can be used
to return within specified time.

Signed-off-by: Vasu Kulkarni <vasu@redhat.com>

diff --git a/teuthology/task/exec.py b/teuthology/task/exec.py
index df66b1a1d87c396238546ddcf7a9aec3a5e8deff..8d1540a6adb39d397052c6d37c15e325af4618e7 100644 (file)
--- a/teuthology/task/exec.py
+++ b/teuthology/task/exec.py
@@ -4,6 +4,7 @@ Exececute custom commands
 import logging
 
 from teuthology import misc as teuthology
+from teuthology.contextutil import safe_while
 
 log = logging.getLogger(__name__)
 
@@ -15,6 +16,9 @@ def task(ctx, config):
         - ceph:
         - kclient: [client.a]
         - exec:
+            retry: 10
+            sleep_for_retry: 5 (in seconds) # defauts to 1 second
+            timeout: 100 (in seconds)
             client.a:
               - "echo 'module libceph +p' > /sys/kernel/debug/dynamic_debug/control"
               - "echo 'module ceph +p' > /sys/kernel/debug/dynamic_debug/control"
@@ -23,7 +27,7 @@ def task(ctx, config):
     It stops and fails with the first command that does not return on success. It means
     that if the first command fails, the second won't run at all.
 
-    To avoid confusion it is recommended to explicitly enclose the commands in 
+    To avoid confusion it is recommended to explicitly enclose the commands in
     double quotes. For instance if the command is false (without double quotes) it will
     be interpreted as a boolean by the YAML parser.
 
@@ -35,6 +39,11 @@ def task(ctx, config):
 
     testdir = teuthology.get_testdir(ctx)
 
+    timeout = config.get('timeout', None)
+    retry = config.get('retry', None)
+    if retry is not None:
+        sleep_for_retry = config.get('sleep_for_retry', 1)
+
     if 'all' in config and len(config) == 1:
         a = config['all']
         roles = teuthology.all_roles(ctx.cluster)
@@ -45,12 +54,30 @@ def task(ctx, config):
         log.info('Running commands on role %s host %s', role, remote.name)
         for c in ls:
             c.replace('$TESTDIR', testdir)
-            remote.run(
-                args=[
-                    'sudo',
-                    'TESTDIR={tdir}'.format(tdir=testdir),
-                    'bash',
-                    '-c',
-                    c],
-                )
-
+            if retry:
+                with safe_while(sleep=sleep_for_retry, tries=retry,
+                                action="exec_with_retry") as proceed:
+                    while proceed():
+                        proc = remote.run(
+                                  args=[
+                                      'sudo',
+                                      'TESTDIR={tdir}'.format(tdir=testdir),
+                                      'bash',
+                                      '-c',
+                                      c],
+                                  timeout=timeout,
+                                  check_status=False,
+                                  wait=True,
+                                 )
+                        if proc.exitstatus == 0:
+                            break
+            else:
+                remote.run(
+                    args=[
+                        'sudo',
+                        'TESTDIR={tdir}'.format(tdir=testdir),
+                        'bash',
+                        '-c',
+                        c],
+                    timeout=timeout
+                    )