admin_socket: loop until the socket command succeeds

Sometimes the thing we're talking to is slow to start, or to register the
command we are running.  Loop in that case, at least for a while.

Signed-off-by: Sage Weil <sage@inktank.com>
(cherry picked from commit 222b296019d8d88d3d2fd3def947658bc076dcac)

Conflicts:
	teuthology/task/admin_socket.py

diff --git a/teuthology/task/admin_socket.py b/teuthology/task/admin_socket.py
index 3bb8296d9dc585639107d300d7e92e24ce577a68..4747a4752c37c646f0993bd6c65515523041b6da 100644 (file)
--- a/teuthology/task/admin_socket.py
+++ b/teuthology/task/admin_socket.py
@@ -68,18 +68,26 @@ def _socket_command(ctx, remote, socket_path, command, args):
     """
     json_fp = StringIO()
     testdir = teuthology.get_testdir(ctx)
-    remote.run(
-        args=[
-            'sudo',
-            '{tdir}/enable-coredump'.format(tdir=testdir),
-            'ceph-coverage',
-            '{tdir}/archive/coverage'.format(tdir=testdir),
-            'ceph',
-            '--admin-daemon', socket_path,
-            command,
-            ] + args,
-        stdout=json_fp,
-        )
+    max_tries = 60
+    while True:
+        proc = remote.run(
+            args=[
+                'sudo',
+                '{tdir}/enable-coredump'.format(tdir=testdir),
+                'ceph-coverage',
+                '{tdir}/archive/coverage'.format(tdir=testdir),
+                'ceph',
+                '--admin-daemon', socket_path,
+                command,
+                ] + args,
+            stdout=json_fp,
+            )
+        if proc.exitstatus == 0:
+            break
+        assert max_tries > 0
+        max_tries -= 1
+        log.info('ceph cli returned an error, command not registered yet?  sleeping and retrying ...')
+        time.sleep(1)
     out = json_fp.getvalue()
     json_fp.close()
     log.debug('admin socket command %s returned %s', command, out)