From 196be4a1290f1546b1e4f50d12955a316cde3521 Mon Sep 17 00:00:00 2001
From: Vasu Kulkarni <vasu@redhat.com>
Date: Thu, 4 Jan 2018 17:51:04 -0800
Subject: [PATCH] Add option to retry for exec task.

retry and sleep_for_retry options can be used with exec task without
needing to use lengthy shell based while loops for similar
task. Also adds timeout option for commands that can be used
to return within specified time.

Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
---
 teuthology/task/exec.py | 47 ++++++++++++++++++++++++++++++++---------
 1 file changed, 37 insertions(+), 10 deletions(-)

diff --git a/teuthology/task/exec.py b/teuthology/task/exec.py
index df66b1a1d8..8d1540a6ad 100644
--- a/teuthology/task/exec.py
+++ b/teuthology/task/exec.py
@@ -4,6 +4,7 @@ Exececute custom commands
 import logging
 
 from teuthology import misc as teuthology
+from teuthology.contextutil import safe_while
 
 log = logging.getLogger(__name__)
 
@@ -15,6 +16,9 @@ def task(ctx, config):
         - ceph:
         - kclient: [client.a]
         - exec:
+            retry: 10
+            sleep_for_retry: 5 (in seconds) # defauts to 1 second
+            timeout: 100 (in seconds)
             client.a:
               - "echo 'module libceph +p' > /sys/kernel/debug/dynamic_debug/control"
               - "echo 'module ceph +p' > /sys/kernel/debug/dynamic_debug/control"
@@ -23,7 +27,7 @@ def task(ctx, config):
     It stops and fails with the first command that does not return on success. It means
     that if the first command fails, the second won't run at all.
 
-    To avoid confusion it is recommended to explicitly enclose the commands in 
+    To avoid confusion it is recommended to explicitly enclose the commands in
     double quotes. For instance if the command is false (without double quotes) it will
     be interpreted as a boolean by the YAML parser.
 
@@ -35,6 +39,11 @@ def task(ctx, config):
 
     testdir = teuthology.get_testdir(ctx)
 
+    timeout = config.get('timeout', None)
+    retry = config.get('retry', None)
+    if retry is not None:
+        sleep_for_retry = config.get('sleep_for_retry', 1)
+
     if 'all' in config and len(config) == 1:
         a = config['all']
         roles = teuthology.all_roles(ctx.cluster)
@@ -45,12 +54,30 @@ def task(ctx, config):
         log.info('Running commands on role %s host %s', role, remote.name)
         for c in ls:
             c.replace('$TESTDIR', testdir)
-            remote.run(
-                args=[
-                    'sudo',
-                    'TESTDIR={tdir}'.format(tdir=testdir),
-                    'bash',
-                    '-c',
-                    c],
-                )
-
+            if retry:
+                with safe_while(sleep=sleep_for_retry, tries=retry,
+                                action="exec_with_retry") as proceed:
+                    while proceed():
+                        proc = remote.run(
+                                  args=[
+                                      'sudo',
+                                      'TESTDIR={tdir}'.format(tdir=testdir),
+                                      'bash',
+                                      '-c',
+                                      c],
+                                  timeout=timeout,
+                                  check_status=False,
+                                  wait=True,
+                                 )
+                        if proc.exitstatus == 0:
+                            break
+            else:
+                remote.run(
+                    args=[
+                        'sudo',
+                        'TESTDIR={tdir}'.format(tdir=testdir),
+                        'bash',
+                        '-c',
+                        c],
+                    timeout=timeout
+                    )
-- 
2.39.5