From 4adc1da58ae2da172b790f939582d9da4d184bd0 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Fri, 2 Aug 2013 20:28:46 -0700 Subject: [PATCH] radosgw-admin: various multiregion related fixes and changes Go to the master zone in the master region for radosgw-admin operations. Trigger metadata sync. Other fixes. Signed-off-by: Yehuda Sadeh --- teuthology/task/radosgw-admin.py | 32 ++++++++++++++---- teuthology/task/radosgw-agent.py | 7 ++++ teuthology/task_util/rgw.py | 57 ++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 7 deletions(-) diff --git a/teuthology/task/radosgw-admin.py b/teuthology/task/radosgw-admin.py index 6b6e5cbf91b6e..090c4fec39c72 100644 --- a/teuthology/task/radosgw-admin.py +++ b/teuthology/task/radosgw-admin.py @@ -12,6 +12,8 @@ import boto.exception import boto.s3.connection import boto.s3.acl +import teuthology.task_util.rgw as rgw_utils + import time from teuthology import misc as teuthology @@ -41,8 +43,11 @@ def task(ctx, config): config = dict.fromkeys(config) clients = config.keys() - # just use the first client... - client = clients[0]; + client = rgw_utils.get_master_client(ctx, clients) + + if not client: + # oh, well, just use the first client... multiregion stuff might not work correctly + client = clients[0]; ## user1='foo' @@ -215,6 +220,8 @@ def task(ctx, config): assert not err assert len(out) == 0 + rgw_utils.radosgw_agent_sync_all(ctx) + # connect to rgw (remote,) = ctx.cluster.only(client).remotes.iterkeys() (remote_user, remote_host) = remote.name.split('@') @@ -367,9 +374,13 @@ def task(ctx, config): for obj in out: # TESTCASE 'log-show','log','show','after activity','returns expected info' + if obj[:4] == 'meta' or obj[:4] == 'data': + continue + (err, log) = rgwadmin(ctx, client, ['log', 'show', '--object', obj]) assert not err assert len(log) > 0 + assert log['bucket'].find(bucket_name) == 0 assert log['bucket'] != bucket_name or log['bucket_id'] == bucket_id assert log['bucket_owner'] == user1 or log['bucket'] == bucket_name + '5' @@ -563,12 +574,19 @@ def task(ctx, config): assert err # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule' + # + (err, out) = rgwadmin(ctx, client, ['zone', 'get']) - assert len(out) > 0 - assert len(out['placement_pools']) == 1 + orig_placement_pools = len(out['placement_pools']) + + # removed this test, it is not correct to assume that zone has default placement, it really + # depends on how we set it up before + # + # assert len(out) > 0 + # assert len(out['placement_pools']) == 1 - default_rule = out['placement_pools'][0] - assert default_rule['key'] == 'default-placement' + # default_rule = out['placement_pools'][0] + # assert default_rule['key'] == 'default-placement' rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}} @@ -579,4 +597,4 @@ def task(ctx, config): (err, out) = rgwadmin(ctx, client, ['zone', 'get']) assert len(out) > 0 - assert len(out['placement_pools']) == 2 + assert len(out['placement_pools']) == orig_placement_pools + 1 diff --git a/teuthology/task/radosgw-agent.py b/teuthology/task/radosgw-agent.py index 62e6025672ff6..40920ff6af779 100644 --- a/teuthology/task/radosgw-agent.py +++ b/teuthology/task/radosgw-agent.py @@ -1,5 +1,6 @@ import contextlib import logging +import argparse from ..orchestra import run from teuthology import misc as teuthology @@ -108,9 +109,15 @@ def task(ctx, config): assert isinstance(config, dict), 'rgw_sync_agent requires a dictionary config' log.debug("config is %s", config) + + ctx.radosgw_agent = argparse.Namespace() + ctx.radosgw_agent.config = config + procs = [(client, run_radosgw_agent(ctx, client, c_config)) for client, c_config in config.iteritems()] + ctx.radosgw_agent.procs = procs + try: yield finally: diff --git a/teuthology/task_util/rgw.py b/teuthology/task_util/rgw.py index c737e29795f7e..3dca2ccd2ba53 100644 --- a/teuthology/task_util/rgw.py +++ b/teuthology/task_util/rgw.py @@ -1,6 +1,7 @@ from cStringIO import StringIO import logging import json +import requests from urlparse import urlparse from teuthology import misc as teuthology @@ -17,8 +18,10 @@ def rgwadmin(ctx, client, cmd, stdin=StringIO(), check_status=False): 'radosgw-admin'.format(tdir=testdir), '--log-to-stderr', '--format', 'json', + '-n', client, ] pre.extend(cmd) + log.info('radosgw-admin: cmd=%s' % pre) (remote,) = ctx.cluster.only(client).remotes.iterkeys() proc = remote.run( args=pre, @@ -53,6 +56,35 @@ def get_zone_host_and_port(ctx, client, zone): return host, port assert False, 'no endpoint for zone {zone} found'.format(zone=zone) +def get_master_zone(ctx, client): + _, region_map = rgwadmin(ctx, client, check_status=True, + cmd=['-n', client, 'region-map', 'get']) + regions = region_map['regions'] + for region in regions: + is_master = (region['val']['is_master'] == "true") + log.info('region={r} is_master={ism}'.format(r=region, ism=is_master)) + if not is_master: + continue + master_zone = region['val']['master_zone'] + log.info('master_zone=%s' % master_zone) + for zone_info in region['val']['zones']: + if zone_info['name'] == master_zone: + return master_zone + log.info('couldn\'t find master zone') + return None + +def get_master_client(ctx, clients): + master_zone = get_master_zone(ctx, clients[0]) # can use any client for this as long as system configured correctly + if not master_zone: + return None + + for client in clients: + zone = zone_for_client(ctx, client) + if zone == master_zone: + return client + + return None + def get_zone_system_keys(ctx, client, zone): _, zone_info = rgwadmin(ctx, client, check_status=True, cmd=['-n', client, @@ -65,3 +97,28 @@ def zone_for_client(ctx, client): ceph_config.update(ctx.ceph.conf.get('client', {})) ceph_config.update(ctx.ceph.conf.get(client, {})) return ceph_config.get('rgw zone') + + +def radosgw_agent_sync(ctx, agent_host, agent_port): + print 'agent_host', agent_host, 'port', agent_port + log.info('sync agent {h}:{p}'.format(h=agent_host, p=agent_port)) + return requests.post('http://{addr}:{port}/metadata/incremental'.format(addr = agent_host, port = agent_port)) + +def radosgw_agent_sync_all(ctx): + if ctx.radosgw_agent.procs: + for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): + dest_zone = zone_for_client(ctx, agent_client) + port = c_config.get('port', 8000) + dest_host, dest_port = get_zone_host_and_port(ctx, agent_client, dest_zone) + radosgw_agent_sync(ctx, dest_host, port) + +def radosgw_agent_sync_all(ctx): + if ctx.radosgw_agent.procs: + for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): + dest_zone = zone_for_client(ctx, agent_client) + port = c_config.get('port', 8000) + dest_host, dest_port = get_zone_host_and_port(ctx, agent_client, dest_zone) + radosgw_agent_sync(ctx, dest_host, port) + + + -- 2.39.5