From 69bbafa804993d2a6cec608ac4f2eb4bfbb01753 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Tue, 15 Nov 2016 11:24:25 -0500 Subject: [PATCH] rgw: add retry/backoff to sync agent requests resolves an issue where startup of the radosgw-agent races with the requests we send to it to run sync. uses the requests package with urllib3 to add retry with backoff to these requests Fixes: http://tracker.ceph.com/issues/16129 Signed-off-by: Casey Bodley --- tasks/util/rgw.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tasks/util/rgw.py b/tasks/util/rgw.py index fe03311c9aa..46160bcaad2 100644 --- a/tasks/util/rgw.py +++ b/tasks/util/rgw.py @@ -2,6 +2,7 @@ from cStringIO import StringIO import logging import json import requests +from requests.packages.urllib3.util import Retry from urlparse import urlparse from teuthology.orchestra.connection import split_user @@ -138,13 +139,21 @@ def radosgw_data_log_window(ctx, client): def radosgw_agent_sync_data(ctx, agent_host, agent_port, full=False): log.info('sync agent {h}:{p}'.format(h=agent_host, p=agent_port)) + # use retry with backoff to tolerate slow startup of radosgw-agent + s = requests.Session() + s.mount('http://{addr}:{port}/'.format(addr = agent_host, port = agent_port), + requests.adapters.HTTPAdapter(max_retries=Retry(total=5, backoff_factor=1))) method = "full" if full else "incremental" - return requests.post('http://{addr}:{port}/data/{method}'.format(addr = agent_host, port = agent_port, method = method)) + return s.post('http://{addr}:{port}/data/{method}'.format(addr = agent_host, port = agent_port, method = method)) def radosgw_agent_sync_metadata(ctx, agent_host, agent_port, full=False): log.info('sync agent {h}:{p}'.format(h=agent_host, p=agent_port)) + # use retry with backoff to tolerate slow startup of radosgw-agent + s = requests.Session() + s.mount('http://{addr}:{port}/'.format(addr = agent_host, port = agent_port), + requests.adapters.HTTPAdapter(max_retries=Retry(total=5, backoff_factor=1))) method = "full" if full else "incremental" - return requests.post('http://{addr}:{port}/metadata/{method}'.format(addr = agent_host, port = agent_port, method = method)) + return s.post('http://{addr}:{port}/metadata/{method}'.format(addr = agent_host, port = agent_port, method = method)) def radosgw_agent_sync_all(ctx, full=False, data=False): if ctx.radosgw_agent.procs: -- 2.39.5