From 5e6538e623c0dea46203cd0c74201b7700f4767c Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Tue, 15 Nov 2016 13:44:27 -0500 Subject: [PATCH] rgw: start_rgw() polls gateway until it accepts connections resolves various races between radosgw startup and further operations - both within the rgw task itself (such as the 'radosgw-admin realm pull'), and in later tasks Fixes: http://tracker.ceph.com/issues/17794 Fixes: http://tracker.ceph.com/issues/17872 Signed-off-by: Casey Bodley --- tasks/rgw.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tasks/rgw.py b/tasks/rgw.py index a1a25f9f84e7e..c1194e6116f10 100644 --- a/tasks/rgw.py +++ b/tasks/rgw.py @@ -10,6 +10,9 @@ import time import errno import util.rgw as rgw_utils +from requests.packages.urllib3 import PoolManager +from requests.packages.urllib3.util import Retry + from cStringIO import StringIO from teuthology.orchestra import run @@ -353,6 +356,17 @@ def start_rgw(ctx, config, on_client = None, except_client = None): wait=False, ) + # XXX: add_daemon() doesn't let us wait until radosgw finishes startup + # use a connection pool with retry/backoff to poll each gateway until it starts listening + http = PoolManager(retries=Retry(connect=8, backoff_factor=1)) + for client in clients_to_run: + if client == except_client: + continue + host, port = ctx.rgw.role_endpoints[client] + endpoint = 'http://{host}:{port}/'.format(host=host, port=port) + log.info('Polling {client} until it starts accepting connections on {endpoint}'.format(client=client, endpoint=endpoint)) + http.request('GET', endpoint) + try: yield finally: -- 2.39.5