From: Sage Weil <sage@inktank.com>
Date: Fri, 19 Jul 2013 22:10:43 +0000 (-0700)
Subject: admin_socket: loop until the socket command succeeds
X-Git-Tag: 1.1.0~2061
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=222b296019d8d88d3d2fd3def947658bc076dcac;p=teuthology.git

admin_socket: loop until the socket command succeeds

Sometimes the thing we're talking to is slow to start, or to register the
command we are running.  Loop in that case, at least for a while.

Signed-off-by: Sage Weil <sage@inktank.com>
---

diff --git a/teuthology/task/admin_socket.py b/teuthology/task/admin_socket.py
index 53799f311d..7b800eef4b 100644
--- a/teuthology/task/admin_socket.py
+++ b/teuthology/task/admin_socket.py
@@ -68,18 +68,26 @@ def _socket_command(ctx, remote, socket_path, command, args):
     """
     json_fp = StringIO()
     testdir = teuthology.get_testdir(ctx)
-    remote.run(
-        args=[
-            'sudo',
-            '{tdir}/adjust-ulimits'.format(tdir=testdir),
-            'ceph-coverage',
-            '{tdir}/archive/coverage'.format(tdir=testdir),
-            'ceph',
-            '--admin-daemon', socket_path,
-            command,
-            ] + args,
-        stdout=json_fp,
-        )
+    max_tries = 60
+    while True:
+        proc = remote.run(
+            args=[
+                'sudo',
+                '{tdir}/adjust-ulimits'.format(tdir=testdir),
+                'ceph-coverage',
+                '{tdir}/archive/coverage'.format(tdir=testdir),
+                'ceph',
+                '--admin-daemon', socket_path,
+                command,
+                ] + args,
+            stdout=json_fp,
+            )
+        if proc.exitstatus == 0:
+            break
+        assert max_tries > 0
+        max_tries -= 1
+        log.info('ceph cli returned an error, command not registered yet?  sleeping and retrying ...')
+        time.sleep(1)
     out = json_fp.getvalue()
     json_fp.close()
     log.debug('admin socket command %s returned %s', command, out)