From 189e04fa8838b9f7b222fdb0c0ab46a2543dbdd7 Mon Sep 17 00:00:00 2001 From: Josh Durgin Date: Thu, 1 Aug 2013 17:18:00 -0700 Subject: [PATCH] rgw_sync_agent: fix a bunch of issues - Read ceph.conf from stored copy that includes overrides - Get system users and keys from cluster instead of reading other tasks' yaml, which may not be complete. - Put zone info extraction from the cluster into utility functions, since it'll be useful for other tests later. - Work with more than one agent on a single host - Accept more than one client to run, like almost every other task - Rename target to dest for consistency with radosgw-agent - Don't make everything one large function Signed-off-by: Josh Durgin --- teuthology/task/rgw_sync_agent.py | 209 ++++++++++++++++-------------- teuthology/task_util/rgw.py | 22 ++++ 2 files changed, 132 insertions(+), 99 deletions(-) diff --git a/teuthology/task/rgw_sync_agent.py b/teuthology/task/rgw_sync_agent.py index ebd985d3ed..b33e5c483b 100644 --- a/teuthology/task/rgw_sync_agent.py +++ b/teuthology/task/rgw_sync_agent.py @@ -1,127 +1,138 @@ import contextlib -import json +from copy import deepcopy import logging from ..orchestra import run from teuthology import misc as teuthology -from teuthology.task_util.rgw import rgwadmin +import teuthology.task_util.rgw as rgw_utils log = logging.getLogger(__name__) -def find_task_config(ctx, name): - for entry in ctx.config['tasks']: - log.info("checking entry: %s", entry) - if name in entry: - return entry[name] - return dict() - -@contextlib.contextmanager -def task(ctx, config): +def run_radosgw_agent(ctx, client, config): """ - Turn on a radosgw sync agent in test mode. Specify: - host: to run on, or leave it blank for client.0 - source: the source region and zone - target: the target region and zone - - tasks: - - ceph: - - rgw: - - rgw_sync_agent: - host: client.0 - source: client.0 - target: client.1 + Run a single radosgw-agent. See task() for config format. """ + src_client = config['src'] + dest_client = config['dest'] - assert config is not None, "rgw_sync_agent requires a config" - if not 'host' in config: - log.info("setting blank host to be client.0") - config['host'] = "client.0" - - log.info("config is %s", config) - - ceph_conf = find_task_config(ctx, 'ceph')['conf'] - log.info("ceph_conf is %s", ceph_conf) - - source = config['source'] - target = config['target'] - - log.info("source is %s", source) - log.info("target is %s", target) - source_region_name = ceph_conf[source]['rgw region'] - target_region_name = ceph_conf[target]['rgw region'] - - rgw_conf = find_task_config(ctx, 'rgw') - log.info("rgw_conf is %s", rgw_conf) - source_system_user = rgw_conf[source]['system user']['name'] - source_access_key = rgw_conf[source]['system user']['access key'] - source_secret_key = rgw_conf[source]['system user']['secret key'] - target_system_user = rgw_conf[target]['system user']['name'] - target_access_key = rgw_conf[target]['system user']['access key'] - target_secret_key = rgw_conf[target]['system user']['secret key'] - - (sync_host,) = ctx.cluster.only(config['host']).remotes - (source_host,) = ctx.cluster.only(source).remotes - - (error, source_region_json) = rgwadmin(ctx, source, - cmd=['-n', source, 'region', 'get', - '--rgw-region', source_region_name]) - log.info("got source_region_json %s", source_region_json) - - (source_host,) = source_region_json['endpoints'] - source_zone = source_region_json['master_zone'] + ceph_config = ctx.ceph.conf.get('global', {}) + ceph_config.update(ctx.ceph.conf.get('client', {})) + src_ceph_config = deepcopy(ceph_config) + src_ceph_config.update(ctx.ceph.conf.get(src_client, {})) + dest_ceph_config = deepcopy(ceph_config) + dest_ceph_config.update(ctx.ceph.conf.get(dest_client, {})) - (error, target_region_json) = rgwadmin(ctx, target, - cmd=['-n', target, 'region', 'get', - '--rgw-region', target_region_name]) - - (target_host,) = target_region_json['endpoints'] - target_zone = target_region_json['master_zone'] - - log.info("got target_region_json %s", target_region_json) + src_zone = src_ceph_config['rgw zone'] + dest_zone = dest_ceph_config['rgw zone'] + log.info("source is %s", src_zone) + log.info("dest is %s", dest_zone) testdir = teuthology.get_testdir(ctx) - - sync_host.run( + (remote,) = ctx.cluster.only(client).remotes.keys() + remote.run( args=[ 'cd', testdir, run.Raw('&&'), - 'git', 'clone', 'https://github.com/ceph/radosgw-agent.git', run.Raw('&&'), - 'cd', "radosgw-agent", run.Raw('&&'), - './bootstrap' + 'git', 'clone', 'https://github.com/ceph/radosgw-agent.git', + 'radosgw-agent.{client}'.format(client=client), + run.Raw('&&'), + 'cd', 'radosgw-agent.{client}'.format(client=client), + run.Raw('&&'), + './bootstrap', ] ) - sync_proc = sync_host.run( + src_host, src_port = rgw_utils.get_zone_host_and_port(ctx, src_client, + src_zone) + dest_host, dest_port = rgw_utils.get_zone_host_and_port(ctx, dest_client, + dest_zone) + src_access, src_secret = rgw_utils.get_zone_system_keys(ctx, src_client, + src_zone) + dest_access, dest_secret = rgw_utils.get_zone_system_keys(ctx, dest_client, + dest_zone) + port = config.get('port', 8000) + daemon_name = '{host}.syncdaemon'.format(host=remote.name) + + return remote.run( args=[ '{tdir}/daemon-helper'.format(tdir=testdir), 'kill', - '{tdir}/radosgw-agent/radosgw-agent'.format(tdir=testdir), - '--src-access-key', source_access_key, - '--src-secret-key', source_secret_key, - '--src-host', source_host, '--src-zone', source_zone, - '--dest-access-key', target_access_key, - '--dest-secret-key', target_secret_key, - '--dest-host', target_host, '--dest-zone', target_zone, - '--daemon-id', '{host}.syncdaemon'.format(host=sync_host.name), - '--test-server-host', 'localhost', '--test-server-port', '8181', - '--log-file', '{tdir}/archive/rgw_sync_agent.log'.format(tdir=testdir) + '{tdir}/radosgw-agent.{client}/radosgw-agent'.format(tdir=testdir, + client=client), + '--src-access-key', src_access, + '--src-secret-key', src_secret, + '--src-host', src_host, + '--src-port', str(src_port), + '--src-zone', src_zone, + '--dest-access-key', dest_access, + '--dest-secret-key', dest_secret, + '--dest-host', dest_host, + '--dest-port', str(dest_port), + '--dest-zone', dest_zone, + '--daemon-id', daemon_name, + '--test-server-host', '0.0.0.0', '--test-server-port', str(port), + '--log-file', '{tdir}/archive/rgw_sync_agent.{client}.log'.format( + tdir=testdir, + client=client), ], wait=False, stdin=run.PIPE, - logger=log.getChild(config['host']) + logger=log.getChild(daemon_name) ) - - yield - + + +@contextlib.contextmanager +def task(ctx, config): + """ + Run radosgw-agents in test mode. + + Configuration is clients to run the agents on, with settings for + source client, destination client, and port to listen on. Binds + to 0.0.0.0. Port defaults to 8000. This must be run on clients + that have the correct zone root pools and rgw zone set in + ceph.conf, or the task cannot read the region information from the + cluster. An example:: + + tasks: + - ceph: + conf: + client.0: + rgw zone = foo + rgw zone root pool = .root.pool + client.1: + rgw zone = bar + rgw zone root pool = .root.pool2 + - rgw: # region configuration omitted for brevity + - rgw_sync_agent: + client.0: + src: client.0 + dest: client.1 + # port: 8000 (default) + client.1: + src: client.1 + dest: client.0 + port: 8001 + """ + assert isinstance(config, dict), 'rgw_sync_agent requires a dictionary config' + log.debug("config is %s", config) + + procs = [(client, run_radosgw_agent(ctx, client, c_config)) for + client, c_config in config.iteritems()] + try: - log.info("shutting down sync agent") - sync_proc.stdin.close() - log.info("waiting on sync agent") - sync_proc.exitstatus.get() + yield finally: - log.info("cleaning up sync agent directory") - sync_host.run( - args=[ - 'rm', '-r', '{tdir}/radosgw-agent'.format(tdir=testdir) - ] - ) - + testdir = teuthology.get_testdir(ctx) + try: + for client, proc in procs: + log.info("shutting down sync agent on %s", client) + proc.stdin.close() + proc.exitstatus.get() + finally: + for client, proc in procs: + ctx.cluster.only(client).run( + args=[ + 'rm', '-rf', + '{tdir}/radosgw-agent.{client}'.format(tdir=testdir, + client=client) + ] + ) diff --git a/teuthology/task_util/rgw.py b/teuthology/task_util/rgw.py index 43bbdcf9ff..8a0c87e54b 100644 --- a/teuthology/task_util/rgw.py +++ b/teuthology/task_util/rgw.py @@ -1,6 +1,7 @@ from cStringIO import StringIO import logging import json +from urlparse import urlparse from teuthology import misc as teuthology @@ -37,3 +38,24 @@ def rgwadmin(ctx, client, cmd, stdin=StringIO(), check_status=False): j = out log.info(' raw result: %s' % j) return (r, j) + +def get_zone_host_and_port(ctx, client, zone): + _, region_map = rgwadmin(ctx, client, check_status=True, + cmd=['-n', client, 'region-map', 'get']) + regions = region_map['regions'] + for region in regions: + for zone_info in region['val']['zones']: + if zone_info['name'] == zone: + endpoint = urlparse(zone_info['endpoints'][0]) + host, port = endpoint.hostname, endpoint.port + if port is None: + port = 80 + return host, port + assert False, 'no endpoint for zone {zone} found'.format(zone=zone) + +def get_zone_system_keys(ctx, client, zone): + _, zone_info = rgwadmin(ctx, client, check_status=True, + cmd=['-n', client, + 'zone', 'get', '--rgw-zone', zone]) + system_key = zone_info['system_key'] + return system_key['access_key'], system_key['secret_key'] -- 2.39.5