admin_socket: loop until the socket command succeeds

Sometimes the thing we're talking to is slow to start, or to register the
command we are running.  Loop in that case, at least for a while.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/teuthology/task/admin_socket.py b/teuthology/task/admin_socket.py
index 53799f311db09436c0b42801e98f4c0d037c9aad..7b800eef4b4b8cc20397758f0f2a2aca69960a00 100644 (file)
--- a/teuthology/task/admin_socket.py
+++ b/teuthology/task/admin_socket.py
@@ -68,18 +68,26 @@ def _socket_command(ctx, remote, socket_path, command, args):
     """
     json_fp = StringIO()
     testdir = teuthology.get_testdir(ctx)
-    remote.run(
-        args=[
-            'sudo',
-            '{tdir}/adjust-ulimits'.format(tdir=testdir),
-            'ceph-coverage',
-            '{tdir}/archive/coverage'.format(tdir=testdir),
-            'ceph',
-            '--admin-daemon', socket_path,
-            command,
-            ] + args,
-        stdout=json_fp,
-        )
+    max_tries = 60
+    while True:
+        proc = remote.run(
+            args=[
+                'sudo',
+                '{tdir}/adjust-ulimits'.format(tdir=testdir),
+                'ceph-coverage',
+                '{tdir}/archive/coverage'.format(tdir=testdir),
+                'ceph',
+                '--admin-daemon', socket_path,
+                command,
+                ] + args,
+            stdout=json_fp,
+            )
+        if proc.exitstatus == 0:
+            break
+        assert max_tries > 0
+        max_tries -= 1
+        log.info('ceph cli returned an error, command not registered yet?  sleeping and retrying ...')
+        time.sleep(1)
     out = json_fp.getvalue()
     json_fp.close()
     log.debug('admin socket command %s returned %s', command, out)