From 88cab4712fe77a3870fa3200263376e3c323cc70 Mon Sep 17 00:00:00 2001 From: Joe Buck Date: Sat, 13 Jul 2013 11:28:57 -0700 Subject: [PATCH] teuth: reworked rgw to support regions / zones Signed-off-by: Joe Buck --- teuthology/task/rgw.py | 519 +++++++++++++++++++++++++++++------------ 1 file changed, 372 insertions(+), 147 deletions(-) diff --git a/teuthology/task/rgw.py b/teuthology/task/rgw.py index 2d534caadf5da..c1d3d784aa566 100644 --- a/teuthology/task/rgw.py +++ b/teuthology/task/rgw.py @@ -8,9 +8,58 @@ from cStringIO import StringIO from teuthology import misc as teuthology from teuthology import contextutil from ..orchestra import run +import ceph_manager log = logging.getLogger(__name__) +# this was lifted from radosgw-admin-rest. +def rgwadmin(ctx, remote, cmd): + log.info('radosgw-admin: %s' % cmd) + testdir = teuthology.get_testdir(ctx) + pre = [ + '{tdir}/adjust-ulimits'.format(tdir=testdir), + 'ceph-coverage'.format(tdir=testdir), + '{tdir}/archive/coverage'.format(tdir=testdir), + 'radosgw-admin'.format(tdir=testdir), + '--log-to-stderr', + '--format', 'json', + ] + pre.extend(cmd) + proc = remote.run( + args=pre, + check_status=False, + stdout=StringIO(), + stderr=StringIO(), + ) + r = proc.exitstatus + out = proc.stdout.getvalue() + j = None + if not r and out != '': + try: + j = json.loads(out) + log.info(' json result: %s' % j) + except ValueError: + j = out + log.info(' raw result: %s' % j) + return (r, j) + +# this was lifted from lost_unfound.py +def rados(ctx, remote, cmd): + testdir = teuthology.get_testdir(ctx) + log.info("rados %s" % ' '.join(cmd)) + pre = [ + '{tdir}/adjust-ulimits'.format(tdir=testdir), + 'ceph-coverage', + '{tdir}/archive/coverage'.format(tdir=testdir), + 'rados', + ]; + pre.extend(cmd) + proc = remote.run( + args=pre, + check_status=False + ) + + return proc.exitstatus @contextlib.contextmanager def create_dirs(ctx, config): @@ -118,7 +167,8 @@ def start_rgw(ctx, config): if client_config is None: client_config = {} log.info("rgw %s config is %s", client, client_config) - + id_ = client.split('.', 1)[1] + log.info('client {client} is id {id}'.format(client=client, id=id_)) run_cmd=[ 'sudo', '{tdir}/adjust-ulimits'.format(tdir=testdir), @@ -130,8 +180,9 @@ def start_rgw(ctx, config): run_cmd_tail=[ 'radosgw', # authenticate as the client this is co-located with - '-i', '{client}'.format(client=client), - '-k', '/etc/ceph/ceph.keyring', + '--name', '{client}'.format(client=client), + # use the keyring for the client we're running as + '-k', '/etc/ceph/ceph.{client}.keyring'.format(client=client), '--log-file', '/var/log/ceph/rgw.log', '--rgw_ops_log_socket_path', '{tdir}/rgw.opslog.sock'.format(tdir=testdir), '{tdir}/apache/apache.conf'.format(tdir=testdir), @@ -212,12 +263,119 @@ def start_apache(ctx, config): run.wait(apaches.itervalues()) +# pulls the configured zone info out and also +# inserts some sensible defaults. +def extract_zone_info(zone_info): + + # now send the zone settings across the wire + system_user = zone_info['user'] + system_access_key = zone_info['access_key'] + system_secret_key = zone_info['secret_key'] + zone_suffix = zone_info['zone_suffix'] + + # new dict to hold the data + zone_dict = {} + zone_dict['domain_root'] = '.rgw.root' + zone_suffix + zone_dict['control_pool'] = '.rgw.control' + zone_suffix + zone_dict['gc_pool'] = '.rgw.gc' + zone_suffix + zone_dict['log_pool'] = '.log' + zone_suffix + zone_dict['intent_log_pool'] = '.intent-log' + zone_suffix + zone_dict['usage_log_pool'] = '.usage' + zone_suffix + zone_dict['user_keys_pool'] = '.users' + zone_suffix + zone_dict['user_email_pool'] = '.users.email' + zone_suffix + zone_dict['user_swift_pool'] = '.users.swift' + zone_suffix + zone_dict['user_uid_pool'] = '.users.id' + zone_suffix + + system_user_dict = {} + system_user_dict['user'] = system_user + system_user_dict['access_key'] = system_access_key + system_user_dict['secret_key'] = system_secret_key + + zone_dict['system_key'] = system_user_dict + + return zone_dict + +# pulls the configured region info out and also +# inserts some sensible defaults. +def extract_region_info(region_info, host_name): + # create a new, empty dict to populate + region_dict = {} + # use specifed region name or default _'region' + if region_info.has_key('name'): + region_dict['name'] = region_info['name'] + else: + region_dict['name'] = '{client}_region'.format(client=client) + + # use specified api name or default to 'default' + if region_info.has_key('api_name'): + region_dict['api_name'] = region_info['api_name'] + else: + region_dict['api_name'] = 'default' + + # Going to assume this is specified for now + if region_info.has_key('is_master'): + region_dict['is_master'] = region_info['is_master'] + else: + pass + #region_dict['is_master'] = 'false' + + # use specified api name or default to 'default' + if region_info.has_key('api_name'): + region_dict['api_name'] = region_info['api_name'] + else: + region_dict['api_name'] = 'default' + + # add in sensible defaults for the endpoints field + if region_info.has_key('endpoints'): + region_dict['endpoints'] = region_info['endpoints'] + else: + region_dict['endpoints'] = [] + region_dict['endpoints'].append('http://{client}:80/'.format(client=host_name)) + + # use specified master zone or default to the first specified zone + # (if there's more than one) + if region_info.has_key('master_zone'): + region_dict['master_zone'] = region_info['master_zone'] + else: + region_dict['master_zone'] = 'default' + + region_dict['zones'] = [] + zones = region_info['zones'].split(',') + for zone in zones: + # build up a zone + name, log_meta, log_data = zone.split(':') + log.info('zone: {zone} meta:{meta} data:{data}'.format(zone=name, + meta=log_meta, data=log_data)) + new_zone_dict = {} + new_zone_dict['name'] = name + new_zone_dict['endpoints'] = [] + new_zone_dict['endpoints'].append('http://{client}:80/'.format(client=host_name)) + new_zone_dict['log_meta'] = log_meta + new_zone_dict['log_data'] = log_data + region_dict['zones'].append(new_zone_dict) + + # just using defaults for now, revisit this later to allow + # the configs to specify placement_targets and default_placement policies + region_dict['placement_targets'] = [] + default_placement_dict = {} + default_placement_dict['name'] = 'default-placement' + default_placement_dict['tags'] = [] + region_dict['placement_targets'].append(default_placement_dict) + + region_dict['default_placement'] = 'default-placement' + + return region_dict + @contextlib.contextmanager def configure_regions_and_zones(ctx, config): log.info('Configuring regions and zones...') - # track files to delete, just easier than reproducing host/filepaths - files_to_delete = {} + client_list = [] + region_files = [] + zone_files = [] + + # iterate through all the clients, creating files with the region zone info and shipping those + # to all hosts. Then, pull out the zone info and stash it in client_list for client in config.iterkeys(): (remote,) = ctx.cluster.only(client).remotes.iterkeys() @@ -230,178 +388,245 @@ def configure_regions_and_zones(ctx, config): log.info("rgw %s config is %s dns is %s", client, client_config, host_name) - files_to_delete[remote] = [] - if 'region_info' in client_config: - region_info = client_config['region_info'] - log.info('region info for {client}: {data}'.format(client=client, data=region_info)) - - # create a new, empty dict to populate - region_dict = {} - # use specifed region name or default _'region' - if region_info.has_key('name'): - region_dict['name'] = region_info['name'] - else: - region_dict['name'] = '{client}_region'.format(client=client) - - # use specified api name or default to 'default' - if region_info.has_key('api_name'): - region_dict['api_name'] = region_info['api_name'] - else: - region_dict['api_name'] = 'default' - - # I don't think we should make assumptions here. - # Going to assume this is specified for now - if region_info.has_key('is_master'): - region_dict['is_master'] = region_info['is_master'] - else: - pass - #region_dict['is_master'] = 'false' - - # use specified api name or default to 'default' - if region_info.has_key('api_name'): - region_dict['api_name'] = region_info['api_name'] - else: - region_dict['api_name'] = 'default' - - # add in sensible defaults for the endpoints field - if region_info.has_key('endpoints'): - region_dict['endpoints'] = region_info['endpoints'] - else: - region_dict['endpoints'] = [] - region_dict['endpoints'].append('http:\/\/{client}:80\/'.format(client=host_name)) - - # use specified master zone or default to the first specified zone - # (if there's more than one) - if region_info.has_key('master_zone'): - region_dict['master_zone'] = region_info['master_zone'] - else: - region_dict['master_zone'] = 'default' - - region_dict['zones'] = [] - zones = region_info['zones'].split(',') - for zone in zones: - # build up a zone - name, log_meta, log_data = zone.split(':') - log.info('zone: {zone} meta:{meta} data:{data}'.format(zone=name, meta=log_meta, data=log_data)) - new_zone_dict = {} - new_zone_dict['name'] = name - new_zone_dict['endpoints'] = [] - #end_point_list = [] - new_zone_dict['endpoints'].append('http://{client}:80/'.format(client=host_name)) - new_zone_dict['log_meta'] = log_meta - new_zone_dict['log_data'] = log_data - region_dict['zones'].append(new_zone_dict) - - # just using defaults for now, revisit this later to allow - # the configs to specify placement_targets and default_placement policies - region_dict['placement_targets'] = [] - default_placement_dict = {} - default_placement_dict['name'] = 'default-placement' - default_placement_dict['tags'] = [] - region_dict['placement_targets'].append(default_placement_dict) - - region_dict['default_placement'] = 'default-placement' + region_dict = extract_region_info(client_config['region_info'], host_name) + region_name = region_dict['name'] log.info('constructed region info: {data}'.format(data=region_dict)) file_name = region_dict['name'] + '.input' testdir = teuthology.get_testdir(ctx) region_file_path = os.path.join(testdir, file_name) - log.info('Shipping {file_out} to host {host}'.format(file_out=region_file_path, \ - host=host_name)) tmpFile = StringIO() - tmpFile.write('{data}'.format(data=json.dumps(region_dict, sort_keys=True, \ + # use json.dumps as a pretty printer so that the generated file + # is easier to read + tmpFile.write('{data}'.format(data=json.dumps(region_dict, sort_keys=True, indent=4))) - tmpFile.seek(0) - #(remote,) = ctx.cluster.only(client).remotes.keys() - teuthology.write_file( - remote=remote, - path=region_file_path, - data=tmpFile, - ) + # ship this region file to all clients + for region_client in config.iterkeys(): + (region_remote,) = ctx.cluster.only(region_client).remotes.iterkeys() - # add this file to the dictionary of files to be deleted - files_to_delete[remote].append(region_file_path) + log.info('sending region file {region_file} to {host}'.format( \ + region_file=region_file_path, + host=str(region_remote))) - else: # if 'region_info' in client_config: - log.info('no region info found for client {client}'.format(client=client)) - # now work on the zone info - if 'zone_info' in client_config: - zone_info = client_config['zone_info'] - # now send the zone settings across the wire - system_user = zone_info['user'] - system_access_key = zone_info['access_key'] - system_secret_key = zone_info['secret_key'] - zone_suffix = zone_info['zone_suffix'] - log.info('jb:\n\tuser:{user}\n\taccess:{access}\n\tsecret:{secret} \ - \n\tsuffix:{suffix}'.format(user=system_user, access=system_access_key, \ - secret=system_secret_key, suffix=zone_suffix)) - - # new dict to hold the data - zone_dict = {} - zone_dict['domain_root'] = '.rgw.root' + zone_suffix - zone_dict['control_pool'] = '.rgw.control' + zone_suffix - zone_dict['gc_pool'] = '.rgw.gc' + zone_suffix - zone_dict['log_pool'] = '.log' + zone_suffix - zone_dict['intent_log_pool'] = '.intent-log' + zone_suffix - zone_dict['usage_log_pool'] = '.usage' + zone_suffix - zone_dict['user_keys_pool'] = '.users' + zone_suffix - zone_dict['user_email_pool'] = '.users.email' + zone_suffix - zone_dict['user_swift_pool'] = '.users.swift' + zone_suffix - - system_user_dict = {} - system_user_dict['user'] = system_user - system_user_dict['access_key'] = system_access_key - system_user_dict['secret_key'] = system_secret_key - - zone_dict['system_key'] = system_user_dict - - log.info('constructed zone info: {data}'.format(data=zone_dict)) - - file_name = 'zone' + zone_suffix + '.input' - testdir = teuthology.get_testdir(ctx) - zone_file_path = os.path.join(testdir, file_name) - log.info('Shipping {file_out} to host {host}'.format(file_out=zone_file_path, \ - host=host_name)) + tmpFile.seek(0) - tmpFile = StringIO() + teuthology.write_file( + remote=region_remote, + path=region_file_path, + data=tmpFile, + ) - tmpFile.write('{data}'.format(data=zone_dict)) - tmpFile.seek(0) - #(remote,) = ctx.cluster.only(client).remotes.keys() - teuthology.write_file( - remote=remote, - path=zone_file_path, - data=tmpFile, - ) + region_files.append(region_file_path) + + + # now work on the zone info + if 'zone_info' in client_config: + zone_info = client_config['zone_info'] + zone_dict = extract_zone_info(zone_info) + + zone_suffix = zone_info['zone_suffix'] + zone_name = zone_info['name'] + + file_name = 'zone' + zone_suffix + '.input' + testdir = teuthology.get_testdir(ctx) + zone_file_path = os.path.join(testdir, file_name) + log.info('Sending zone file {file_out} to host {host}'.format(file_out=zone_file_path, \ + host=host_name)) + + tmpFile = StringIO() + + tmpFile.write('{data}'.format(data=json.dumps(zone_dict, + sort_keys=True, indent=4))) + + # zone info only gets sent to the particular client + tmpFile.seek(0) + teuthology.write_file( + remote=remote, + path=zone_file_path, + data=tmpFile, + ) + + zone_files.append(zone_file_path) + + # add an entry that is client, remote, region, zone, suffix, zone file path + # this data is used later + client_entry = [] + client_entry.append(client) + client_entry.append(remote) + client_entry.append(region_name) + client_entry.append(zone_name) + client_entry.append(zone_suffix) + client_entry.append(zone_file_path) + client_list.append(client_entry) + + else: # if 'zone_info' in client_config: + log.info('no zone info found for client {client}'.format(client=client)) + + else: # if 'region_info' in client_config: + log.info('no region info found for client {client}'.format(client=client)) - files_to_delete[remote].append(zone_file_path) - else: - log.info('no zone info found for client {client}'.format(client=client)) + # now apply the region info on all clients + for region_file_path in region_files: + for client_entry in client_list: + + client = client_entry[0] + remote = client_entry[1] + region_name = client_entry[2] + zone_name = client_entry[3] + suffix = client_entry[4] + zone_file_path = client_entry[5] + + log.info('cl: {client_name} rn: {region} zn: {zone} rf: {rf} suf:{suffix} zfp:{zfp}'.format(\ + client_name=str(remote), region=region_name, zone=zone_name, + rf=region_file_path, suffix=suffix, zfp=zone_file_path)) + + # now use this file to set the region info on the host + (err, out) = rgwadmin(ctx, remote, ['region', 'set', '--infile', + region_file_path, '--name', + '{client}'.format(client=client), + '--rgw-region', + '{region}'.format(region=region_name), + '--rgw-zone', + '{zone}'.format(zone=zone_name)]) + assert not err + + # go through the clients and set the zone info + for client_entry in client_list: + client = client_entry[0] + remote = client_entry[1] + region_name = client_entry[2] + zone_name = client_entry[3] + suffix = client_entry[4] + zone_file_path = client_entry[5] + + + (err, out) = rgwadmin(ctx, remote, ['zone', 'set', + '--rgw-zone={zone}'.format(zone=zone_name), + '--infile', zone_file_path, + '--name', '{client}'.format(client=client), + '--rgw-region', + '{region}'.format(region=region_name), + ]) + assert not err + + # set this to the default region + for client_entry in client_list: + + client = client_entry[0] + remote = client_entry[1] + region_name = client_entry[2] + zone_name = client_entry[3] + suffix = client_entry[4] + zone_file_path = client_entry[5] + log.info('rf: {rf}'.format(rf=region_file_path)) + (err, out) = rgwadmin(ctx, remote, ['region', 'default', + '--name', '{client}'.format(client=client), + '--rgw-region', + '{region}'.format(region=region_name), + #'--rgw-zone', + #'{zone}'.format(zone=zone_name) + ]) + if out: + log.info('JB out: {data}'.format(data=out())) + if err: + log.info('err: {err} out: {out}'.format(err=err, out=out)) + + assert not err + + # only do this if there's at least one zone specified + if len(client_list) > 0: + # need the mon to make rados calls + first_mon = teuthology.get_first_mon(ctx, config) + (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys() + + # delete the pools for the default + err = rados(ctx, mon, ['-p', '.rgw.root', 'rm', 'region_info.default']) + log.info('err is %d' % err) + + err = rados(ctx, mon, ['-p', '.rgw.root', 'rm', 'zone_info.default']) + log.info('err is %d' % err) + + # delete the pools for the suffixs + for client_entry in client_list: + + client = client_entry[0] + remote = client_entry[1] + region_name = client_entry[2] + zone_name = client_entry[3] + suffix = client_entry[4] + zone_file_path = client_entry[5] + + err = rados(ctx, mon, ['-p', '.rgw.root' + suffix, + 'rm', 'region_info.default']) + log.info('err is %d' % err) + + err = rados(ctx, mon, ['-p', '.rgw.root' + suffix, + 'rm', 'zone_info.default']) + log.info('err is %d' % err) + + # update the region map + for client_entry in client_list: + + client = client_entry[0] + remote = client_entry[1] + region_name = client_entry[2] + zone_name = client_entry[3] + suffix = client_entry[4] + zone_file_path = client_entry[5] + + (err, out) = rgwadmin(ctx, remote, ['region-map', 'update', + '--name', + '{client}'.format(client=client), + '--rgw-region', + '{region}'.format(region=region_name), + '--rgw-zone', + '{zone}'.format(zone=zone_name)]) + assert not err try: yield finally: log.info('Cleaning up regions and zones....') - for remote in files_to_delete.keys(): - per_host_files_to_delete = files_to_delete[remote] - for to_delete in per_host_files_to_delete: - log.info('deleting {file_name} from host {host}'.format(file_name=to_delete, \ - host=str(remote))) - ctx.cluster.only(remote).run( + + for client_entry in client_list: + + client = client_entry[0] + remote = client_entry[1] + region_name = client_entry[2] + zone_name = client_entry[3] + suffix = client_entry[4] + zone_file_path = client_entry[5] + + for region_file in region_files: + log.info('deleting {file_name} from host {host}'.format( \ + file_name=region_file, + host=str(remote))) + remote.run( args=[ 'rm', '-f', - '{file_path}'.format(file_path=to_delete), + '{file_path}'.format(file_path=region_file), ], ) + for zone_file in zone_files: + log.info('deleting {file_name} from host {host}'.format( \ + file_name=zone_file, + host=str(remote))) + remote.run( + args=[ + 'rm', + '-f', + '{file_path}'.format(file_path=zone_file), + ], + ) @contextlib.contextmanager def task(ctx, config): -- 2.39.5