rgw: start_rgw() polls gateway until it accepts connections

resolves various races between radosgw startup and further operations -
both within the rgw task itself (such as the 'radosgw-admin realm pull'),
and in later tasks

Fixes: http://tracker.ceph.com/issues/17794
Fixes: http://tracker.ceph.com/issues/17872
Signed-off-by: Casey Bodley <cbodley@redhat.com>

diff --git a/tasks/rgw.py b/tasks/rgw.py
index a1a25f9f84e7ecfa31729cb2d57c99d4b9143ada..c1194e6116f1017280f87d4c7ea71c25ddd794ad 100644 (file)
--- a/tasks/rgw.py
+++ b/tasks/rgw.py
@@ -10,6 +10,9 @@ import time
 import errno
 import util.rgw as rgw_utils
 
+from requests.packages.urllib3 import PoolManager
+from requests.packages.urllib3.util import Retry
+
 from cStringIO import StringIO
 
 from teuthology.orchestra import run
@@ -353,6 +356,17 @@ def start_rgw(ctx, config, on_client = None, except_client = None):
             wait=False,
             )
 
+    # XXX: add_daemon() doesn't let us wait until radosgw finishes startup
+    # use a connection pool with retry/backoff to poll each gateway until it starts listening
+    http = PoolManager(retries=Retry(connect=8, backoff_factor=1))
+    for client in clients_to_run:
+        if client == except_client:
+            continue
+        host, port = ctx.rgw.role_endpoints[client]
+        endpoint = 'http://{host}:{port}/'.format(host=host, port=port)
+        log.info('Polling {client} until it starts accepting connections on {endpoint}'.format(client=client, endpoint=endpoint))
+        http.request('GET', endpoint)
+
     try:
         yield
     finally: